diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml
index 6402f01..86f419e 100644
--- a/.forgejo/workflows/ci.yml
+++ b/.forgejo/workflows/ci.yml
@@ -72,17 +72,28 @@ jobs:
- run: npm run test:unit
- run: npm run build
- integration:
- # This act_runner (swarm-runner v0.6.1) puts service containers on the
- # default bridge with NO service-name DNS, and publishing fixed host
- # ports collides with the operator's running docker-compose dev stack on
- # the same shared daemon. Workaround: publish NO host ports, and reach
- # each service by its bridge IP — discovered at runtime via the mounted
- # docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
- # containers can talk by IP (only embedded DNS is missing), so IP
- # addressing is reliable here. Everything runs in ONE step so resolved
- # values don't depend on cross-step env passing. Pattern documented in
- # FabledRulebook/forgejo.md "CI philosophy".
+ # Integration suite split into THREE parallel shards (2026-05-25, runner
+ # capacity bumped 2→6). Each shard gets its own Postgres + Redis service
+ # set and runs alembic + a disjoint subset of integration tests. Shards
+ # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
+ # stays single-threaded per shard but multiple shards run in parallel
+ # wall-clock. Approximate split — rebalance once --durations=15 output
+ # reveals which shard is the long pole.
+ #
+ # Each shard's docker-ps filter uses its own unique job name to scope
+ # service-container resolution. act_runner appears to strip underscores
+ # from job names when building container labels — `int_api` yielded
+ # zero matches on 2026-05-25 — so shards use no-separator names
+ # (`intapi`, `intimp`, `intcore`) instead. Each step prints
+ # `docker ps -a` first so a future naming-convention shift surfaces in
+ # the log without another guess-and-push cycle.
+ #
+ # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
+ # done — per ci-requirements.md, FC is the only Python consumer of that
+ # image and the CI-Runner project's "add deps to image when used by >1
+ # project" rule keeps the install per-job.
+
+ intapi:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
@@ -113,7 +124,6 @@ jobs:
--health-retries 10
steps:
- uses: actions/checkout@v4
-
- name: Cache pip wheels
uses: actions/cache@v4
with:
@@ -121,15 +131,14 @@ jobs:
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
-
- - name: Integration suite (resolve service IPs, migrate, test)
+ - name: API integration shard (resolve service IPs, migrate, test)
run: |
set -eux
- # Scope to THIS job's service containers (act_runner names them
- # ...JOB-integration...); the operator's compose stack uses the
- # same images but different names, so it won't match.
- PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
- RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
+ echo "=== container landscape (diagnostic for filter scoping) ==="
+ docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+ echo "=== end landscape ==="
+ PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+ RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
test -n "$PG" && test -n "$RD"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
@@ -137,16 +146,153 @@ jobs:
export DB_HOST="$PG_IP"
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
- # Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
for i in $(seq 1 60); do
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
sleep 2
done
- # uv when available (5-10x faster wheel resolve); fall back to pip.
if command -v uv >/dev/null 2>&1; then
uv pip install --system -r requirements.txt pytest pytest-asyncio
else
pip install -r requirements.txt pytest pytest-asyncio
fi
alembic upgrade head
- pytest tests/ -v -m integration --durations=25
+ pytest tests/test_api_*.py -v -m integration --durations=15
+
+ intimp:
+ runs-on: python-ci
+ container:
+ image: git.fabledsword.com/bvandeusen/ci-python:3.14
+ env:
+ DB_USER: fabledcurator
+ DB_PASSWORD: ci_integration
+ DB_PORT: "5432"
+ DB_NAME: fabledcurator_test
+ SECRET_KEY: ci_integration_placeholder
+ services:
+ postgres:
+ image: pgvector/pgvector:pg16
+ env:
+ POSTGRES_USER: fabledcurator
+ POSTGRES_PASSWORD: ci_integration
+ POSTGRES_DB: fabledcurator_test
+ options: >-
+ --health-cmd "pg_isready -U fabledcurator"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 10
+ redis:
+ image: redis:7-alpine
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 10
+ steps:
+ - uses: actions/checkout@v4
+ - name: Cache pip wheels
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pip
+ key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+ restore-keys: |
+ pip-${{ runner.os }}-py314-
+ - name: Importer integration shard (resolve service IPs, migrate, test)
+ run: |
+ set -eux
+ echo "=== container landscape (diagnostic for filter scoping) ==="
+ docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+ echo "=== end landscape ==="
+ PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+ RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
+ test -n "$PG" && test -n "$RD"
+ PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+ RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+ test -n "$PG_IP" && test -n "$RD_IP"
+ export DB_HOST="$PG_IP"
+ export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+ export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+ for i in $(seq 1 60); do
+ (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+ sleep 2
+ done
+ if command -v uv >/dev/null 2>&1; then
+ uv pip install --system -r requirements.txt pytest pytest-asyncio
+ else
+ pip install -r requirements.txt pytest pytest-asyncio
+ fi
+ alembic upgrade head
+ pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
+
+ intcore:
+ runs-on: python-ci
+ container:
+ image: git.fabledsword.com/bvandeusen/ci-python:3.14
+ env:
+ DB_USER: fabledcurator
+ DB_PASSWORD: ci_integration
+ DB_PORT: "5432"
+ DB_NAME: fabledcurator_test
+ SECRET_KEY: ci_integration_placeholder
+ services:
+ postgres:
+ image: pgvector/pgvector:pg16
+ env:
+ POSTGRES_USER: fabledcurator
+ POSTGRES_PASSWORD: ci_integration
+ POSTGRES_DB: fabledcurator_test
+ options: >-
+ --health-cmd "pg_isready -U fabledcurator"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 10
+ redis:
+ image: redis:7-alpine
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 10
+ steps:
+ - uses: actions/checkout@v4
+ - name: Cache pip wheels
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pip
+ key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+ restore-keys: |
+ pip-${{ runner.os }}-py314-
+ - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
+ run: |
+ set -eux
+ echo "=== container landscape (diagnostic for filter scoping) ==="
+ docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+ echo "=== end landscape ==="
+ PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+ RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
+ test -n "$PG" && test -n "$RD"
+ PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+ RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+ test -n "$PG_IP" && test -n "$RD_IP"
+ export DB_HOST="$PG_IP"
+ export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+ export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+ for i in $(seq 1 60); do
+ (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+ sleep 2
+ done
+ if command -v uv >/dev/null 2>&1; then
+ uv pip install --system -r requirements.txt pytest pytest-asyncio
+ else
+ pip install -r requirements.txt pytest pytest-asyncio
+ fi
+ alembic upgrade head
+ pytest tests/ -v -m integration --durations=15 \
+ --ignore-glob='tests/test_api_*.py' \
+ --ignore-glob='tests/test_importer*.py' \
+ --ignore-glob='tests/test_import_*.py' \
+ --ignore-glob='tests/test_migration_*.py' \
+ --ignore-glob='tests/test_phash_*.py' \
+ --ignore-glob='tests/test_sidecar_*.py' \
+ --ignore-glob='tests/test_scan_*.py' \
+ --ignore-glob='tests/test_archive_extractor.py' \
+ --ignore-glob='tests/test_backfill_phash.py'
diff --git a/.forgejo/workflows/extension.yml b/.forgejo/workflows/extension.yml
index dc54c95..ef007c3 100644
--- a/.forgejo/workflows/extension.yml
+++ b/.forgejo/workflows/extension.yml
@@ -2,7 +2,14 @@ name: extension
on:
push:
branches: [dev, main]
- paths: ['extension/**']
+ # Self-retriggering: include the workflow file itself so edits to
+ # this workflow (filename-glob fixes, env tweaks, etc.) run the
+ # sign-and-publish dance on next push without needing a manual
+ # Forgejo dispatch. Side-effect commits (`ext: publish signed XPI`)
+ # only touch frontend/public/extension/ so they DON'T re-trigger.
+ paths:
+ - 'extension/**'
+ - '.forgejo/workflows/extension.yml'
workflow_dispatch:
jobs:
@@ -38,34 +45,52 @@ jobs:
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
- name: Commit signed XPI to frontend/public/extension/
run: |
- set -e
+ set -ex
# AMO renames signed XPIs using its internal addon-id-safe-string
- # (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. The
- # original 'fabledcurator-*' glob never matched and the step
- # exited 1 even on a successful sign (operator-flagged
- # 2026-05-25). Match any *.xpi in the artifacts dir — fresh
- # CI runner means there's exactly one — and rename it on copy
- # so the FC server's whitelist (backend/app/frontend.py expects
- # 'fabledcurator-*.xpi') keeps working.
+ # (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. Match
+ # any *.xpi in the artifacts dir — fresh CI runner means there's
+ # exactly one — and rename it on copy so the FC server's
+ # whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
+ # keeps working.
+ # Diagnostic instrumentation 2026-05-25: a prior run (id 1731)
+ # reported success without producing an `ext: publish signed XPI`
+ # commit on main. Tracing the cwd, artifacts dir, version
+ # extraction, staging state, and push response so the next run's
+ # log explains the gap.
+ echo "=== cwd ==="
+ pwd
+ echo "=== ref / branch ==="
+ echo "ref: ${GITHUB_REF:-unset} sha: ${GITHUB_SHA:-unset}"
+ git log --oneline -3 || true
+ echo "=== artifacts dir ==="
+ ls -la extension/web-ext-artifacts/ 2>&1 || echo "(dir missing)"
XPI=$(ls extension/web-ext-artifacts/*.xpi 2>/dev/null | head -1)
+ echo "XPI=$XPI"
if [ -z "$XPI" ]; then
echo "No XPI produced by web-ext sign — exiting"
- ls -la extension/web-ext-artifacts/ || true
exit 1
fi
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
+ echo "VERSION=$VERSION"
DEST="frontend/public/extension/fabledcurator-${VERSION}.xpi"
+ echo "DEST=$DEST"
mkdir -p frontend/public/extension
# Wipe any prior versions so the directory doesn't grow each release.
rm -f frontend/public/extension/fabledcurator-*.xpi
cp "$XPI" "$DEST"
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
+ echo "=== frontend/public/extension/ after copy ==="
+ ls -la frontend/public/extension/
git config user.name "FC extension CI"
git config user.email "noreply@fabledsword.com"
git add frontend/public/extension/
+ echo "=== git status after add ==="
+ git status --short
+ echo "=== diff --cached --stat ==="
+ git diff --cached --stat || true
if git diff --cached --quiet; then
- echo "No changes to commit"
+ echo "No changes to commit (frontend/public/extension/ matches HEAD)"
else
git commit -m "ext: publish signed XPI fabledcurator-${VERSION}.xpi"
- git push origin HEAD:main
+ git push -v origin HEAD:main
fi
diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py
index f9eb62b..cf4c0c4 100644
--- a/backend/app/services/importer.py
+++ b/backend/app/services/importer.py
@@ -71,6 +71,31 @@ def is_video(path: Path) -> bool:
return path.suffix.lower() in VIDEO_EXTS
+def _safe_ext(path: Path) -> str:
+ """Conservatively extract a file extension for PostAttachment.ext
+ (varchar(32)).
+
+ gallery-dl produces some filenames with URL-encoded query-string
+ artifacts embedded into the basename (e.g.
+ `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
+ `Path.suffix` finds the LAST dot and returns everything after, which
+ in those cases yields a 50+ char "extension" of mostly base64-ish
+ junk. That blows the column. Operator-flagged 2026-05-25.
+
+ Real extensions are short and alphanumeric. We accept anything ≤ 16
+ chars where every post-dot character is alphanumeric; anything else
+ means the input wasn't a real extension and we return the empty
+ string. ext is nullable-ish (empty string still satisfies NOT NULL)
+ and consumers should treat "" as "no known extension".
+ """
+ suffix = path.suffix.lower()
+ if not suffix or len(suffix) > 16:
+ return ""
+ if not all(c.isalnum() for c in suffix[1:]):
+ return ""
+ return suffix
+
+
def _mime_for(path: Path) -> str:
suffix = path.suffix.lower()
image_mimes = {
@@ -209,7 +234,7 @@ class Importer:
sha256=sha,
path=stored,
original_filename=source.name,
- ext=source.suffix.lower(),
+ ext=_safe_ext(source),
mime=_mime_for(source),
size_bytes=source.stat().st_size,
))
diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue
index e1391df..0f39bac 100644
--- a/frontend/src/components/settings/MaintenancePanel.vue
+++ b/frontend/src/components/settings/MaintenancePanel.vue
@@ -14,7 +14,6 @@
-
@@ -27,7 +26,6 @@ import AllowlistTable from './AllowlistTable.vue'
import AliasTable from './AliasTable.vue'
import BackupCard from './BackupCard.vue'
import TagMaintenanceCard from './TagMaintenanceCard.vue'
-import BrowserExtensionCard from './BrowserExtensionCard.vue'
import LegacyMigrationCard from './LegacyMigrationCard.vue'
diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue
index 00a7180..8d41770 100644
--- a/frontend/src/views/SettingsView.vue
+++ b/frontend/src/views/SettingsView.vue
@@ -31,6 +31,11 @@
{{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
Go to Import tab
+
+
@@ -62,6 +67,7 @@ import { useImportStore } from '../stores/import.js'
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
+import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
import ImportTaskList from '../components/settings/ImportTaskList.vue'
diff --git a/tests/test_importer_attachments.py b/tests/test_importer_attachments.py
index 06b6083..16ec52e 100644
--- a/tests/test_importer_attachments.py
+++ b/tests/test_importer_attachments.py
@@ -59,3 +59,25 @@ def test_json_sidecar_is_not_attached(importer, import_layout):
select(func.count()).select_from(PostAttachment)
).scalar_one()
assert n == 0
+
+
+def test_mangled_filename_extension_is_sanitized(importer, import_layout):
+ """gallery-dl sometimes URL-encodes a query string into the basename
+ (`...https___www.patreon.com_media-u_Z0F...`). Python's Path.suffix
+ returns 50+ chars of base64-ish junk for those, which blows the
+ PostAttachment.ext varchar(32) column. Operator-flagged 2026-05-25.
+ The importer should record an empty ext rather than crash."""
+ import_root, _ = import_layout
+ f = (
+ import_root / "Alice"
+ / "79507046_media_https___www.patreon.com_media-u_Z0FBQUFBQm5q"
+ )
+ f.parent.mkdir(parents=True, exist_ok=True)
+ f.write_bytes(b"binary blob")
+ r = importer.import_one(f)
+ assert r.status == "attached"
+ att = importer.session.execute(select(PostAttachment)).scalar_one()
+ # Junk "extension" -> stored as empty string (not the 50-char garbage).
+ assert att.ext == ""
+ # original_filename is Text-typed so the full name survives intact.
+ assert att.original_filename.endswith("_Z0FBQUFBQm5q")