Compare commits

...

14 Commits

Author SHA1 Message Date
bvandeusen 3872e1dda9 Merge pull request 'fix(ext-ci): web-ext v8 .cjs config workaround' (#17) from dev into main 2026-05-25 22:49:14 -04:00
bvandeusen 17e19081a2 fix(ext-ci): drop web-ext-config.cjs (v8 mis-parses .cjs configs as if module.exports were a config option) — inline ignore-files on CLI + --no-config-discovery — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 22:46:58 -04:00
bvandeusen 9814f3dbaf Merge pull request 'Release v26.05.25.5 — Extension publish refactor, deep-scan IR-parity, archive-import perf, artist Settings tab' (#16) from dev into main 2026-05-25 22:44:59 -04:00
bvandeusen 770bcf3aa6 feat(artist): tab split (Overview/Settings) so DangerZone is reachable without exhausting the infinite-scroll image grid — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 22:32:07 -04:00
bvandeusen 52d7905c43 perf(importer): cache phash candidates on Importer to fix archive-import soft-timeout (was O(M×N) per-member SELECTs) — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 22:25:32 -04:00
bvandeusen e6ededbe8e feat(deep-scan): IR-parity port — refreshed status + counter, re-queue completed paths in deep mode, honest UX — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 22:16:26 -04:00
bvandeusen c06cbc0abe feat(ci): inline extension sign into build.yml + Forgejo Release Assets as XPI cache (v26.05.25.5) — bump ext to 1.0.2 — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 21:30:11 -04:00
bvandeusen b214460fdb Merge pull request 'Release v26.05.25.4 — importer ext sanitize fix, CI shard split, BrowserExtensionCard on Overview' (#15) from dev into main 2026-05-25 21:11:50 -04:00
bvandeusen ac39509a74 fix(ci): rename shard jobs to no-separator names (intapi/intimp/intcore) + add diagnostic docker ps dump so next bounce surfaces the real act_runner naming convention — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 20:59:17 -04:00
bvandeusen 3531f373ee feat(settings): move BrowserExtensionCard from Maintenance to Overview tab — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 20:33:44 -04:00
bvandeusen 36cc0622cb fix(importer): sanitize PostAttachment.ext to skip mangled gallery-dl URL-encoded basenames (varchar(32) overrun) — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 20:33:44 -04:00
bvandeusen e50f92d900 perf(ci): shard integration suite into 3 parallel jobs (int_api, int_imp, int_core) — newly feasible after act_runner capacity 2→6 — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 20:05:15 -04:00
bvandeusen ba8d9b112d fix(ext-ci): add diagnostic tracing to commit step to surface why run #309 reported success without producing the XPI side-commit — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 20:05:15 -04:00
bvandeusen c451061ca5 fix(ext-ci): self-retrigger workflow on its own edits (path filter includes workflow file) — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> 2026-05-25 18:24:31 -04:00
21 changed files with 767 additions and 208 deletions
+134 -2
View File
@@ -6,18 +6,150 @@ on:
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
# - write:package, read:package (for docker push to git.fabledsword.com)
# - write:release (for future release-cutting workflows)
# - write:release (for ext-<version> release asset cache)
# - write:issue (for future issue-management automation)
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
jobs:
build-web:
# Sign-or-fetch-from-cache: signs the extension via AMO if no ext-<version>
# Forgejo release exists yet, otherwise downloads the cached signed XPI.
# Result is uploaded as an Actions artifact for build-web to consume.
#
# Why this lives in build.yml (not a separate workflow): the merge-commit's
# docker image tagged `:latest` MUST carry the XPI. A separate sign workflow
# racing build.yml leaves `:latest` without the XPI for ~5min (until the
# commit-back triggers another build). Inline ordering eliminates the race.
# Cache strategy: Forgejo Release Assets — picked 2026-05-25 over Generic
# Packages (cleaner API surface) and commit-back-to-side-branch (no extra
# branch to manage). AMO blocks re-signing the same version (returns 409),
# so signing is intentionally one-shot per version bump.
sign-extension:
if: github.ref == 'refs/heads/main'
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
steps:
- uses: actions/checkout@v4
- name: Resolve extension version
id: extver
run: |
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Resolved extension version: $VERSION"
- name: Check Forgejo release-asset cache
id: cache
env:
TOKEN: ${{ secrets.RELEASE_TOKEN }}
run: |
set -eu
VERSION=${{ steps.extver.outputs.version }}
STATUS=$(curl -s -o release.json -w "%{http_code}" \
-H "Authorization: token $TOKEN" \
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
echo "Tag lookup HTTP status: $STATUS"
if [ "$STATUS" = "200" ]; then
ASSET_ID=$(jq -r '.assets[]? | select(.name | test("\\.xpi$")) | .id' release.json | head -1)
if [ -n "$ASSET_ID" ] && [ "$ASSET_ID" != "null" ]; then
echo "cached=true" >> "$GITHUB_OUTPUT"
echo "asset_id=$ASSET_ID" >> "$GITHUB_OUTPUT"
echo "Cached XPI exists at ext-$VERSION (asset id $ASSET_ID); skipping AMO sign"
else
echo "cached=false" >> "$GITHUB_OUTPUT"
echo "Release ext-$VERSION exists but has no .xpi asset; will re-sign + re-upload"
fi
else
echo "cached=false" >> "$GITHUB_OUTPUT"
echo "No release named ext-$VERSION; will sign via AMO and upload"
fi
- name: Download cached signed XPI (cache hit)
if: steps.cache.outputs.cached == 'true'
env:
TOKEN: ${{ secrets.RELEASE_TOKEN }}
run: |
set -eu
mkdir -p extension/web-ext-artifacts
curl -sL -H "Authorization: token $TOKEN" \
-o "extension/web-ext-artifacts/fabledcurator-${{ steps.extver.outputs.version }}.xpi" \
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/assets/${{ steps.cache.outputs.asset_id }}"
ls -la extension/web-ext-artifacts/
- name: Sign via AMO (cache miss)
if: steps.cache.outputs.cached != 'true'
run: |
cd extension && npm install --no-save --no-audit --no-fund && npm run sign
env:
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
- name: Upload signed XPI to ext-<version> release (cache miss)
if: steps.cache.outputs.cached != 'true'
env:
TOKEN: ${{ secrets.RELEASE_TOKEN }}
run: |
set -eux
VERSION=${{ steps.extver.outputs.version }}
# AMO renames signed XPIs with its internal addon-id-safe-string;
# canonicalize to fabledcurator-<version>.xpi so the FC server's
# whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
# keeps working.
SIGNED=$(ls extension/web-ext-artifacts/*.xpi | head -1)
XPI="extension/web-ext-artifacts/fabledcurator-$VERSION.xpi"
cp "$SIGNED" "$XPI"
# Find-or-create the ext-<version> release.
STATUS=$(curl -s -o release.json -w "%{http_code}" \
-H "Authorization: token $TOKEN" \
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
if [ "$STATUS" != "200" ]; then
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
-d "{\"tag_name\":\"ext-$VERSION\",\"name\":\"Extension $VERSION (signed XPI cache)\",\"body\":\"Internal cache for the signed XPI consumed by build.yml's build-web job. Not a user-facing FC release.\",\"target_commitish\":\"main\"}" \
-o release.json \
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases"
fi
RELEASE_ID=$(jq -r '.id' release.json)
test -n "$RELEASE_ID" && test "$RELEASE_ID" != "null"
curl -s -X POST -H "Authorization: token $TOKEN" \
-F "attachment=@$XPI" \
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi"
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
- name: Upload XPI as Actions artifact (handoff to build-web)
uses: actions/upload-artifact@v4
with:
name: signed-extension
path: extension/web-ext-artifacts/fabledcurator-*.xpi
retention-days: 1
build-web:
needs: [sign-extension]
# sign-extension is main-only; on dev it's skipped, build-web still runs.
if: always() && (needs.sign-extension.result == 'success' || needs.sign-extension.result == 'skipped')
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
steps:
- uses: actions/checkout@v4
- name: Download signed extension (main only)
if: github.ref == 'refs/heads/main'
uses: actions/download-artifact@v4
with:
name: signed-extension
path: extension/web-ext-artifacts/
- name: Place signed XPI in build context (main only)
if: github.ref == 'refs/heads/main'
run: |
set -eux
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
XPI=$(ls extension/web-ext-artifacts/*.xpi | head -1)
mkdir -p frontend/public/extension
cp "$XPI" "frontend/public/extension/fabledcurator-$VERSION.xpi"
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
ls -la frontend/public/extension/
- name: Determine tag
id: tag
run: |
+168 -22
View File
@@ -72,17 +72,28 @@ jobs:
- run: npm run test:unit
- run: npm run build
integration:
# This act_runner (swarm-runner v0.6.1) puts service containers on the
# default bridge with NO service-name DNS, and publishing fixed host
# ports collides with the operator's running docker-compose dev stack on
# the same shared daemon. Workaround: publish NO host ports, and reach
# each service by its bridge IP — discovered at runtime via the mounted
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
# containers can talk by IP (only embedded DNS is missing), so IP
# addressing is reliable here. Everything runs in ONE step so resolved
# values don't depend on cross-step env passing. Pattern documented in
# FabledRulebook/forgejo.md "CI philosophy".
# Integration suite split into THREE parallel shards (2026-05-25, runner
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
# set and runs alembic + a disjoint subset of integration tests. Shards
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
# stays single-threaded per shard but multiple shards run in parallel
# wall-clock. Approximate split — rebalance once --durations=15 output
# reveals which shard is the long pole.
#
# Each shard's docker-ps filter uses its own unique job name to scope
# service-container resolution. act_runner appears to strip underscores
# from job names when building container labels — `int_api` yielded
# zero matches on 2026-05-25 — so shards use no-separator names
# (`intapi`, `intimp`, `intcore`) instead. Each step prints
# `docker ps -a` first so a future naming-convention shift surfaces in
# the log without another guess-and-push cycle.
#
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
# done — per ci-requirements.md, FC is the only Python consumer of that
# image and the CI-Runner project's "add deps to image when used by >1
# project" rule keeps the install per-job.
intapi:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
@@ -113,7 +124,6 @@ jobs:
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
uses: actions/cache@v4
with:
@@ -121,15 +131,14 @@ jobs:
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: Integration suite (resolve service IPs, migrate, test)
- name: API integration shard (resolve service IPs, migrate, test)
run: |
set -eux
# Scope to THIS job's service containers (act_runner names them
# ...JOB-integration...); the operator's compose stack uses the
# same images but different names, so it won't match.
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
test -n "$PG" && test -n "$RD"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
@@ -137,16 +146,153 @@ jobs:
export DB_HOST="$PG_IP"
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
for i in $(seq 1 60); do
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
sleep 2
done
# uv when available (5-10x faster wheel resolve); fall back to pip.
if command -v uv >/dev/null 2>&1; then
uv pip install --system -r requirements.txt pytest pytest-asyncio
else
pip install -r requirements.txt pytest pytest-asyncio
fi
alembic upgrade head
pytest tests/ -v -m integration --durations=25
pytest tests/test_api_*.py -v -m integration --durations=15
intimp:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
env:
DB_USER: fabledcurator
DB_PASSWORD: ci_integration
DB_PORT: "5432"
DB_NAME: fabledcurator_test
SECRET_KEY: ci_integration_placeholder
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: fabledcurator
POSTGRES_PASSWORD: ci_integration
POSTGRES_DB: fabledcurator_test
options: >-
--health-cmd "pg_isready -U fabledcurator"
--health-interval 10s
--health-timeout 5s
--health-retries 10
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: Importer integration shard (resolve service IPs, migrate, test)
run: |
set -eux
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
test -n "$PG" && test -n "$RD"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
test -n "$PG_IP" && test -n "$RD_IP"
export DB_HOST="$PG_IP"
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
for i in $(seq 1 60); do
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
sleep 2
done
if command -v uv >/dev/null 2>&1; then
uv pip install --system -r requirements.txt pytest pytest-asyncio
else
pip install -r requirements.txt pytest pytest-asyncio
fi
alembic upgrade head
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
intcore:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
env:
DB_USER: fabledcurator
DB_PASSWORD: ci_integration
DB_PORT: "5432"
DB_NAME: fabledcurator_test
SECRET_KEY: ci_integration_placeholder
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: fabledcurator
POSTGRES_PASSWORD: ci_integration
POSTGRES_DB: fabledcurator_test
options: >-
--health-cmd "pg_isready -U fabledcurator"
--health-interval 10s
--health-timeout 5s
--health-retries 10
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
run: |
set -eux
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
test -n "$PG" && test -n "$RD"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
test -n "$PG_IP" && test -n "$RD_IP"
export DB_HOST="$PG_IP"
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
for i in $(seq 1 60); do
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
sleep 2
done
if command -v uv >/dev/null 2>&1; then
uv pip install --system -r requirements.txt pytest pytest-asyncio
else
pip install -r requirements.txt pytest pytest-asyncio
fi
alembic upgrade head
pytest tests/ -v -m integration --durations=15 \
--ignore-glob='tests/test_api_*.py' \
--ignore-glob='tests/test_importer*.py' \
--ignore-glob='tests/test_import_*.py' \
--ignore-glob='tests/test_migration_*.py' \
--ignore-glob='tests/test_phash_*.py' \
--ignore-glob='tests/test_sidecar_*.py' \
--ignore-glob='tests/test_scan_*.py' \
--ignore-glob='tests/test_archive_extractor.py' \
--ignore-glob='tests/test_backfill_phash.py'
+12 -54
View File
@@ -1,8 +1,19 @@
name: extension
# Lint-only workflow. The sign-and-publish dance moved into build.yml's
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
# because sign-extension runs as a build-web dependency in the SAME workflow,
# eliminating the prior race between build.yml and a separate extension.yml.
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
on:
push:
branches: [dev, main]
paths: ['extension/**']
paths:
- 'extension/**'
- '.forgejo/workflows/extension.yml'
pull_request:
branches: [main]
paths:
- 'extension/**'
workflow_dispatch:
jobs:
@@ -16,56 +27,3 @@ jobs:
run: cd extension && npm install --no-save --no-audit --no-fund
- name: Lint
run: cd extension && npm run lint
sign-and-publish:
needs: lint
if: github.ref == 'refs/heads/main'
runs-on: python-ci
container:
image: node:22-bookworm-slim
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.RELEASE_TOKEN }}
- name: Install web-ext + git
run: |
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
cd extension && npm install --no-save --no-audit --no-fund
- name: Sign XPI
run: cd extension && npm run sign
env:
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
- name: Commit signed XPI to frontend/public/extension/
run: |
set -e
# AMO renames signed XPIs using its internal addon-id-safe-string
# (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. The
# original 'fabledcurator-*' glob never matched and the step
# exited 1 even on a successful sign (operator-flagged
# 2026-05-25). Match any *.xpi in the artifacts dir — fresh
# CI runner means there's exactly one — and rename it on copy
# so the FC server's whitelist (backend/app/frontend.py expects
# 'fabledcurator-*.xpi') keeps working.
XPI=$(ls extension/web-ext-artifacts/*.xpi 2>/dev/null | head -1)
if [ -z "$XPI" ]; then
echo "No XPI produced by web-ext sign — exiting"
ls -la extension/web-ext-artifacts/ || true
exit 1
fi
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
DEST="frontend/public/extension/fabledcurator-${VERSION}.xpi"
mkdir -p frontend/public/extension
# Wipe any prior versions so the directory doesn't grow each release.
rm -f frontend/public/extension/fabledcurator-*.xpi
cp "$XPI" "$DEST"
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
git config user.name "FC extension CI"
git config user.email "noreply@fabledsword.com"
git add frontend/public/extension/
if git diff --cached --quiet; then
echo "No changes to commit"
else
git commit -m "ext: publish signed XPI fabledcurator-${VERSION}.xpi"
git push origin HEAD:main
fi
@@ -0,0 +1,38 @@
"""import_batch.refreshed counter for deep-scan sidecar re-application
Revision ID: 0019
Revises: 0018
Create Date: 2026-05-25
Adds a `refreshed` counter to `import_batch`, mirroring the existing
`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now
re-applies sidecar metadata to already-imported files (the IR feature
that didn't make the FC port the first time); a "refreshed" outcome
increments this counter so the UI can surface "X new, Y refreshed"
instead of the misleading "Scan complete — no new files" message.
server_default=0 backfills existing rows in place — no UPDATE needed.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0019"
down_revision: Union[str, None] = "0018"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"import_batch",
sa.Column(
"refreshed", sa.Integer(),
nullable=False, server_default=sa.text("0"),
),
)
def downgrade() -> None:
op.drop_column("import_batch", "refreshed")
+1
View File
@@ -53,6 +53,7 @@ async def status():
"imported": active.imported,
"skipped": active.skipped,
"failed": active.failed,
"refreshed": active.refreshed,
"started_at": active.started_at.isoformat(),
}
return jsonify(payload)
+4
View File
@@ -26,6 +26,10 @@ class ImportBatch(Base):
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Deep-scan only: count of already-imported files whose sidecar metadata
# got re-applied this run (post/source/provenance upsert). Stays 0 on
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
# running | complete | cancelled
+101 -31
View File
@@ -51,7 +51,14 @@ class SkipReason(StrEnum):
@dataclass(frozen=True)
class ImportResult:
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
# 'imported' — new ImageRecord row created
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
# 'attached' — non-media saved as PostAttachment
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
# artist on an already-imported row (no new ImageRecord)
# 'skipped' — no work done (true duplicate, too small, etc.)
# 'failed' — pipeline error
status: str
image_id: int | None = None
skip_reason: SkipReason | None = None
error: str | None = None
@@ -71,6 +78,31 @@ def is_video(path: Path) -> bool:
return path.suffix.lower() in VIDEO_EXTS
def _safe_ext(path: Path) -> str:
"""Conservatively extract a file extension for PostAttachment.ext
(varchar(32)).
gallery-dl produces some filenames with URL-encoded query-string
artifacts embedded into the basename (e.g.
`79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
`Path.suffix` finds the LAST dot and returns everything after, which
in those cases yields a 50+ char "extension" of mostly base64-ish
junk. That blows the column. Operator-flagged 2026-05-25.
Real extensions are short and alphanumeric. We accept anything ≤ 16
chars where every post-dot character is alphanumeric; anything else
means the input wasn't a real extension and we return the empty
string. ext is nullable-ish (empty string still satisfies NOT NULL)
and consumers should treat "" as "no known extension".
"""
suffix = path.suffix.lower()
if not suffix or len(suffix) > 16:
return ""
if not all(c.isalnum() for c in suffix[1:]):
return ""
return suffix
def _mime_for(path: Path) -> str:
suffix = path.suffix.lower()
image_mimes = {
@@ -126,6 +158,49 @@ class Importer:
self.settings = settings
self.deep = deep
self.attachments = AttachmentStore(images_root)
# phash near-dup candidate cache. Archive imports call _import_media
# per-member; without this cache the per-member SELECT *FROM
# image_record WHERE phash IS NOT NULL fetch repeats N times and a
# large library × many-member archive blew past soft_time_limit
# (300s) — operator-flagged 2026-05-25. Loaded lazily on first
# need, appended to on every imported/superseded outcome, never
# invalidated mid-Importer (Importer instances are per-task /
# per-archive-import so cross-instance staleness is harmless).
self._phash_candidates: list[tuple] | None = None
def _phash_candidates_cache(self) -> list[tuple]:
"""Cached `(phash, width, height, id)` rows from image_record.
Loaded on first call, appended-to on subsequent imported/
superseded outcomes. Soft-timeout pattern: an archive with N
members + a library of M existing rows used to do N × M-row
fetches (operator-flagged 2026-05-25); now it's exactly one.
The per-task lifecycle of Importer (instantiated fresh by
import_media_file) bounds the cache's staleness window: cross-
process changes (other workers importing concurrently) won't
be reflected, but that's the same race the un-cached version
had — `find_similar` is best-effort anyway."""
if self._phash_candidates is None:
rows = self.session.execute(
select(
ImageRecord.phash,
ImageRecord.width,
ImageRecord.height,
ImageRecord.id,
).where(ImageRecord.phash.is_not(None))
).all()
self._phash_candidates = [
(r.phash, r.width or 0, r.height or 0, r.id) for r in rows
]
return self._phash_candidates
def _phash_cache_append(self, phash, width, height, image_id) -> None:
"""Append a freshly-imported row to the cache so subsequent
members of the same archive can match against it."""
if self._phash_candidates is not None and phash is not None:
self._phash_candidates.append(
(phash, width or 0, height or 0, image_id)
)
def import_one(self, source: Path) -> ImportResult:
"""Dispatch by kind. Media → normal pipeline. Archive → extract
@@ -209,7 +284,7 @@ class Importer:
sha256=sha,
path=stored,
original_filename=source.name,
ext=source.suffix.lower(),
ext=_safe_ext(source),
mime=_mime_for(source),
size_bytes=source.stat().st_size,
))
@@ -324,18 +399,7 @@ class Importer:
error=f"PIL load failed during phash compute: {exc}",
)
if phash is not None:
cand_rows = self.session.execute(
select(
ImageRecord.phash,
ImageRecord.width,
ImageRecord.height,
ImageRecord.id,
).where(ImageRecord.phash.is_not(None))
).all()
candidates = [
(c.phash, c.width or 0, c.height or 0, c.id)
for c in cand_rows
]
candidates = self._phash_candidates_cache()
rel, match_id = find_similar(
phash, width or 0, height or 0,
candidates, self.settings.phash_threshold,
@@ -369,6 +433,7 @@ class Importer:
)
self.session.add(record)
self.session.flush()
self._phash_cache_append(phash, width, height, record.id)
# Folder→artist (anchored to attribution_path).
artist = None
@@ -392,13 +457,27 @@ class Importer:
) -> ImportResult:
"""Deep scan: backfill phash/provenance/artist on an
already-imported record. METADATA ONLY — never re-runs the pHash
near-dup / supersede path. NULL-only, idempotent."""
near-dup / supersede path. NULL-only on phash/artist, additive on
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
Idempotent: a second deep-scan over the same file finds nothing
to refresh and is a no-op.
Returns status="refreshed" so the UI can surface the work done
instead of the prior misleading "skipped/duplicate_hash" reading.
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
as a no-op skip during the original port and the UI showed deep
scan as "completed with no changes" even when sidecar metadata
actually got re-applied to N existing rows.
"""
if existing.phash is None and not is_video(source):
try:
with Image.open(source) as im:
ph = compute_phash(im)
if ph is not None:
existing.phash = ph
# Promoted from NULL to non-NULL → cache is now stale
# (this row would newly qualify for the candidates set).
self._phash_candidates = None
except Exception as exc:
log.warning("deep rephash failed for %s: %s", source, exc)
@@ -411,10 +490,7 @@ class Importer:
self._apply_sidecar(existing, attribution_path, artist)
self.session.commit()
return ImportResult(
status="skipped", skip_reason=SkipReason.duplicate_hash,
image_id=existing.id, error="deep: re-derived",
)
return ImportResult(status="refreshed", image_id=existing.id)
def attach_in_place(
self,
@@ -498,18 +574,7 @@ class Importer:
except Exception:
phash = None
if phash is not None:
cand_rows = self.session.execute(
select(
ImageRecord.phash,
ImageRecord.width,
ImageRecord.height,
ImageRecord.id,
).where(ImageRecord.phash.is_not(None))
).all()
candidates = [
(c.phash, c.width or 0, c.height or 0, c.id)
for c in cand_rows
]
candidates = self._phash_candidates_cache()
rel, match_id = find_similar(
phash, width or 0, height or 0,
candidates, self.settings.phash_threshold,
@@ -544,6 +609,7 @@ class Importer:
record.artist_id = artist.id
self.session.add(record)
self.session.flush()
self._phash_cache_append(phash, width, height, record.id)
# Sidecar provenance (best-effort). When `source` is passed, link
# the post to that subscription Source instead of creating a new
@@ -759,6 +825,10 @@ class Importer:
# created_at intentionally preserved; updated_at auto-bumps.
self.session.flush()
self.session.commit()
# The phash candidate cache (used to avoid N+1 selects during
# archive imports) is now stale for `existing.id` — the row's
# phash/dimensions changed. Invalidate; the next call re-fetches.
self._phash_candidates = None
# Sidecar enrichment from the new (larger) file's location.
# _apply_sidecar resolves artist from the sidecar itself if the
+13 -2
View File
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
def _map_result_to_status(result):
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
'superseded' = the kept row's file/ML changed → complete + re-derive.
'attached' = a non-art file preserved → complete, no ML/thumb."""
'attached' = a non-art file preserved → complete, no ML/thumb.
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
complete, no ML/thumb re-derive (file/pixels unchanged)."""
if result.status in ("imported", "superseded"):
return ("complete", True)
if result.status == "attached":
if result.status in ("attached", "refreshed"):
return ("complete", False)
if result.status == "skipped":
return ("skipped", False)
@@ -138,6 +140,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
task.result_image_id = result.image_id
counter_col_name = "imported"
counter_col = ImportBatch.imported
elif result.status == "refreshed":
# Deep-scan rederive: existing row got phash/artist/sidecar
# refreshed. Task is complete (no further work), but counted in
# `refreshed` not `imported` so the UI can surface the actual
# work done. operator-flagged 2026-05-25.
task.status = "complete"
task.result_image_id = result.image_id
counter_col_name = "refreshed"
counter_col = ImportBatch.refreshed
elif result.status == "attached":
task.status = "complete"
counter_col_name = "attachments"
+17 -8
View File
@@ -59,16 +59,25 @@ def scan_directory(self, triggered_by: str = "manual",
session.flush()
batch_id = batch.id
# Skip-set: any source_path that already has a non-failed ImportTask
# row. Re-running scan_directory must not re-enqueue files the
# importer has already handled (or is currently handling); doing so
# creates duplicate work and inflates the queue. Failed prior tasks
# are eligible for retry.
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
#
# quick: any non-failed prior ImportTask (active OR finished) is
# skipped — quick scan only does new-file enqueue, so re-touching
# already-imported files is wasted work.
#
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
# are skipped. Completed and skipped tasks ARE re-queued because
# deep scan exists precisely to re-touch already-imported files
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
active_statuses = ["pending", "queued", "processing"]
if mode == "deep":
skip_statuses = active_statuses
else:
skip_statuses = active_statuses + ["complete", "skipped"]
non_failed_existing = set(session.execute(
select(ImportTask.source_path).where(
ImportTask.status.in_(
["pending", "queued", "processing", "complete", "skipped"]
),
ImportTask.status.in_(skip_statuses),
)
).scalars().all())
+1 -1
View File
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "FabledCurator",
"version": "1.0.1",
"version": "1.0.2",
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
"browser_specific_settings": {
+5 -5
View File
@@ -1,13 +1,13 @@
{
"name": "fabledcurator-extension",
"version": "1.0.1",
"version": "1.0.2",
"private": true,
"description": "Firefox extension for FabledCurator",
"scripts": {
"lint": "web-ext lint --source-dir=.",
"start": "web-ext run --source-dir=. --firefox=firefox",
"build": "web-ext build --source-dir=. --overwrite-dest",
"sign": "web-ext sign --source-dir=. --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
"lint": "web-ext lint --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore",
"start": "web-ext run --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --firefox=firefox",
"build": "web-ext build --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --overwrite-dest",
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
},
"devDependencies": {
"web-ext": "^8.0.0"
-13
View File
@@ -1,13 +0,0 @@
module.exports = {
sourceDir: '.',
artifactsDir: './web-ext-artifacts',
ignoreFiles: [
'package.json',
'package-lock.json',
'web-ext-config.cjs',
'web-ext-artifacts',
'node_modules',
'README.md',
'.gitignore',
],
};
@@ -10,6 +10,9 @@
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
{{ store.activeBatch.source_path || '/import' }}
imported {{ store.activeBatch.imported }},
<template v-if="store.activeBatch.scan_mode === 'deep'">
refreshed {{ store.activeBatch.refreshed || 0 }},
</template>
skipped {{ store.activeBatch.skipped }},
failed {{ store.activeBatch.failed }} /
{{ store.activeBatch.total_files }} files
@@ -26,11 +29,12 @@
<p class="text-body-2 mb-3">
<span v-if="!store.activeBatch">
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
new files only (skips paths already on a non-failed ImportTask).
<strong>Deep scan</strong> additionally chains a phash backfill
across the existing library use after bulk-imports to catch
near-duplicates that slipped through. Both modes route non-media
+ sidecar pairs through PostAttachment capture.
new files only.
<strong>Deep scan</strong> additionally re-walks already-imported
files so updated sidecar metadata (post title/date/attribution) and
previously-NULL phashes / artist links get refreshed. Use after
bulk-downloading fresh sidecars for existing content. Both modes
route non-media + sidecar pairs through PostAttachment capture.
</span>
<span v-else>
An active batch is in progress. Wait for it to finish, or click
@@ -14,7 +14,6 @@
<AliasTable class="mt-4" />
<BackupCard class="mt-6" />
<TagMaintenanceCard class="mt-6" />
<BrowserExtensionCard class="mt-6" />
<LegacyMigrationCard class="mt-6" />
</div>
</template>
@@ -27,7 +26,6 @@ import AllowlistTable from './AllowlistTable.vue'
import AliasTable from './AliasTable.vue'
import BackupCard from './BackupCard.vue'
import TagMaintenanceCard from './TagMaintenanceCard.vue'
import BrowserExtensionCard from './BrowserExtensionCard.vue'
import LegacyMigrationCard from './LegacyMigrationCard.vue'
</script>
+27 -6
View File
@@ -66,21 +66,42 @@ export const useImportStore = defineStore('import', () => {
// batch flashes 'running' for <100ms then 'complete' before the
// first refreshStatus() lands; UI never sees the active state).
const label = mode === 'deep'
? 'Deep scan triggered (pHash backfill chained)'
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
window.__fcToast?.({ text: label, type: 'success' })
await refreshStatus()
// Re-poll twice over ~5s to catch quick-finalize transitions and
// surface a result toast either way. Skip the "no new files" hint
// for 'verify' since it doesn't walk the import_root.
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
// Operator-flagged 2026-05-25: the prior "no new files" message was
// misleading because deep scan IS doing work (refresh) even when
// there are no new files to import. Surface the real workload count
// (imported + refreshed + queued) instead. For quick scan + zero
// queued work, fall back to "up to date" instead of the old
// implementation-detail-leaking message.
setTimeout(async () => {
await refreshStatus()
await loadTasks(true)
if (!activeBatch.value && mode !== 'verify') {
if (activeBatch.value || mode === 'verify') return
// Batch finalized quickly; figure out what actually happened.
// The task list was just refreshed; the freshest row(s) carry
// the batch outcome.
const batchId = tasks.value[0]?.batch_id
const sameBatch = batchId
? tasks.value.filter(t => t.batch_id === batchId)
: []
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
if (mode === 'deep' && sameBatch.length > 0) {
window.__fcToast?.({
text: 'Scan complete — no new files (everything already on an ImportTask row)',
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
type: 'info',
})
} else if (mode === 'quick' && sameBatch.length > 0) {
window.__fcToast?.({
text: `Quick scan finished — ${newImported} new file(s) queued`,
type: 'info',
})
} else {
window.__fcToast?.({ text: 'Library is up to date', type: 'info' })
}
}, 2000)
} catch (e) {
+88 -55
View File
@@ -45,69 +45,92 @@
>Credential health · FC-3b</v-chip>
</div>
<section v-if="store.overview.cooccurring_tags.length" class="fc-artist__sec">
<h2 class="fc-h2">Frequent tags</h2>
<div class="fc-artist__tags">
<v-chip
v-for="t in store.overview.cooccurring_tags" :key="t.id"
size="small" @click="openTag(t.id)"
>{{ t.name }} <span class="fc-artist__tagc">{{ t.count }}</span></v-chip>
</div>
</section>
<!-- Tabs split (2026-05-25): Settings was previously slotted at the
bottom of the page after the infinite-scroll image grid, which
made it effectively unreachable for any artist with more than
a couple of pages of content. The Settings tab now hosts
destructive admin actions (artist+content cascade-delete) and
any future per-artist management UI. v-tabs is `position:
sticky; top: 64px` (under the 64px AppShell TopNav) so it
stays parked while the gallery scrolls. -->
<v-tabs
v-model="tab" color="accent" class="mb-4"
style="position: sticky; top: 64px; z-index: 4;
background: rgb(var(--v-theme-surface));"
>
<v-tab value="overview">Overview</v-tab>
<v-tab value="settings">Settings</v-tab>
</v-tabs>
<section v-if="store.overview.activity.length" class="fc-artist__sec">
<h2 class="fc-h2">Activity</h2>
<svg class="fc-artist__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
preserveAspectRatio="none" role="img" aria-label="posts over time">
<polyline :points="sparkPoints" fill="none"
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
</svg>
</section>
<v-window v-model="tab">
<v-window-item value="overview">
<section v-if="store.overview.cooccurring_tags.length" class="fc-artist__sec">
<h2 class="fc-h2">Frequent tags</h2>
<div class="fc-artist__tags">
<v-chip
v-for="t in store.overview.cooccurring_tags" :key="t.id"
size="small" @click="openTag(t.id)"
>{{ t.name }} <span class="fc-artist__tagc">{{ t.count }}</span></v-chip>
</div>
</section>
<section v-if="store.overview.sources.length" class="fc-artist__sec">
<div class="fc-artist__sec-head">
<h2 class="fc-h2">Sources</h2>
<RouterLink
:to="`/subscriptions?artist_id=${store.overview.id}`"
class="fc-artist__manage"
>Manage subscriptions </RouterLink>
</div>
<v-table density="compact">
<thead>
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
</thead>
<tbody>
<tr v-for="s in store.overview.sources" :key="s.id">
<td>{{ s.platform }}</td>
<td class="fc-artist__url">{{ s.url }}</td>
<td class="text-right">{{ s.image_count }}</td>
</tr>
</tbody>
</v-table>
</section>
<section v-if="store.overview.activity.length" class="fc-artist__sec">
<h2 class="fc-h2">Activity</h2>
<svg class="fc-artist__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
preserveAspectRatio="none" role="img" aria-label="posts over time">
<polyline :points="sparkPoints" fill="none"
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
</svg>
</section>
<section class="fc-artist__sec">
<h2 class="fc-h2">Images</h2>
<MasonryGrid
:items="store.images"
:loading="store.imagesLoading"
:has-more="store.hasMoreImages"
@load-more="store.loadMoreImages(slug)"
@open="openImage"
/>
</section>
<section v-if="store.overview.sources.length" class="fc-artist__sec">
<div class="fc-artist__sec-head">
<h2 class="fc-h2">Sources</h2>
<RouterLink
:to="`/subscriptions?artist_id=${store.overview.id}`"
class="fc-artist__manage"
>Manage subscriptions </RouterLink>
</div>
<v-table density="compact">
<thead>
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
</thead>
<tbody>
<tr v-for="s in store.overview.sources" :key="s.id">
<td>{{ s.platform }}</td>
<td class="fc-artist__url">{{ s.url }}</td>
<td class="text-right">{{ s.image_count }}</td>
</tr>
</tbody>
</v-table>
</section>
<ArtistDangerZone
:slug="slug"
:artist-id="store.overview.id"
:artist-name="store.overview.name"
/>
<section class="fc-artist__sec">
<h2 class="fc-h2">Images</h2>
<MasonryGrid
:items="store.images"
:loading="store.imagesLoading"
:has-more="store.hasMoreImages"
@load-more="store.loadMoreImages(slug)"
@open="openImage"
/>
</section>
</v-window-item>
<v-window-item value="settings">
<ArtistDangerZone
:slug="slug"
:artist-id="store.overview.id"
:artist-name="store.overview.name"
/>
</v-window-item>
</v-window>
</template>
</v-container>
</template>
<script setup>
import { computed, watch } from 'vue'
import { computed, ref, watch } from 'vue'
import { useRoute, useRouter, RouterLink } from 'vue-router'
import { useArtistStore } from '../stores/artist.js'
import { useModalStore } from '../stores/modal.js'
@@ -120,8 +143,18 @@ const store = useArtistStore()
const modal = useModalStore()
const slug = computed(() => route.params.slug)
// Per-artist tab — defaults to Overview. Settings tab hosts destructive
// admin actions (DangerZone). Switching artists resets to Overview so the
// destructive surface isn't re-shown by accident when navigating between
// artists.
const tab = ref('overview')
watch(slug, (s) => { if (s) store.load(s) }, { immediate: true })
watch(slug, (s) => {
if (s) {
store.load(s)
tab.value = 'overview'
}
}, { immediate: true })
const dateRange = computed(() => {
const r = store.overview?.date_range
+6
View File
@@ -31,6 +31,11 @@
{{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
<v-btn variant="text" size="small" @click="tab = 'import'">Go to Import tab</v-btn>
</v-alert>
<!-- Browser-extension install/download lives on Overview (moved
from Maintenance 2026-05-25). Overview is the discovery
surface for "things to set up"; Maintenance is for
housekeeping of already-set-up systems. -->
<BrowserExtensionCard class="mt-6" />
</v-window-item>
<v-window-item value="activity">
@@ -62,6 +67,7 @@ import { useImportStore } from '../stores/import.js'
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
import ImportTaskList from '../components/settings/ImportTaskList.vue'
+22
View File
@@ -59,3 +59,25 @@ def test_json_sidecar_is_not_attached(importer, import_layout):
select(func.count()).select_from(PostAttachment)
).scalar_one()
assert n == 0
def test_mangled_filename_extension_is_sanitized(importer, import_layout):
"""gallery-dl sometimes URL-encodes a query string into the basename
(`...https___www.patreon.com_media-u_Z0F...`). Python's Path.suffix
returns 50+ chars of base64-ish junk for those, which blows the
PostAttachment.ext varchar(32) column. Operator-flagged 2026-05-25.
The importer should record an empty ext rather than crash."""
import_root, _ = import_layout
f = (
import_root / "Alice"
/ "79507046_media_https___www.patreon.com_media-u_Z0FBQUFBQm5q"
)
f.parent.mkdir(parents=True, exist_ok=True)
f.write_bytes(b"binary blob")
r = importer.import_one(f)
assert r.status == "attached"
att = importer.session.execute(select(PostAttachment)).scalar_one()
# Junk "extension" -> stored as empty string (not the 50-char garbage).
assert att.ext == ""
# original_filename is Text-typed so the full name survives intact.
assert att.original_filename.endswith("_Z0FBQUFBQm5q")
+6 -2
View File
@@ -64,9 +64,13 @@ def test_deep_rederives_phash_and_provenance(db_sync, import_layout):
deep = _mk(db_sync, import_layout, deep=True)
r2 = deep.import_one(src)
assert r2.status == "skipped"
assert "deep" in (r2.error or "")
# Outcome flipped from "skipped+duplicate_hash" to "refreshed" 2026-05-25
# so the UI can surface deep scan's actual work instead of showing it as
# a no-op. See ImportResult.status comment + _deep_rederive docstring.
assert r2.status == "refreshed"
assert r2.image_id == rec.id
assert r2.skip_reason is None
assert r2.error is None
db_sync.expire_all()
rec2 = db_sync.get(ImageRecord, rec.id)
+6
View File
@@ -241,3 +241,9 @@ def test_import_task_maps_superseded_to_complete_and_requeues():
assert _map_result_to_status(
ImportResult(status="failed", error="boom")
) == ("failed", False)
# Refreshed (deep scan): complete + no ML/thumb re-derive (pixels
# unchanged). Added 2026-05-25 alongside ImportBatch.refreshed
# counter so deep scan reports "X refreshed" instead of "no work".
assert _map_result_to_status(
ImportResult(status="refreshed", image_id=5)
) == ("complete", False)
+109
View File
@@ -0,0 +1,109 @@
"""Deep scan re-queues already-completed ImportTasks.
Operator-flagged 2026-05-25: deep scan used to skip everything that
already had a non-failed ImportTask row, making a deep re-scan a no-op
when no new files were added. That defeated the entire point of deep
scan (re-apply sidecar metadata to existing rows). The skip-set now
splits by mode — quick keeps the old "any non-failed" semantics; deep
skips ONLY actively-in-flight statuses.
"""
import pytest
from PIL import Image
from sqlalchemy import func, select
from backend.app.models import ImportBatch, ImportSettings, ImportTask
from backend.app.tasks.scan import scan_directory
pytestmark = pytest.mark.integration
def _img(path):
path.parent.mkdir(parents=True, exist_ok=True)
Image.new("RGB", (40, 40), (10, 200, 80)).save(path, "JPEG")
def test_deep_scan_requeues_completed_task(db_sync, tmp_path, monkeypatch):
"""Quick scan then deep scan of the same /import: the file completed
in the first run should be re-enqueued by the deep run (different
ImportTask id, same source_path)."""
import_root = tmp_path / "import"
settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
settings.import_scan_path = str(import_root)
db_sync.commit()
src = import_root / "Mae" / "p.jpg"
_img(src)
from backend.app import celery_app
celery_app.celery.conf.task_always_eager = False # explicit
try:
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
first_task = db_sync.execute(
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
).scalar_one()
# Simulate the worker having finished it.
first_task.status = "complete"
db_sync.commit()
# Now deep scan: the SAME file should get a NEW ImportTask row.
second_batch_id = scan_directory.run(triggered_by="manual", mode="deep")
assert second_batch_id != first_batch_id
new_tasks_in_second_batch = db_sync.execute(
select(func.count())
.select_from(ImportTask)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert new_tasks_in_second_batch == 1, (
"deep scan did not re-queue the completed file"
)
# And the second batch's task should be for the same source_path
# as the first (proves it's a re-queue, not a different file).
sp = db_sync.execute(
select(ImportTask.source_path)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert sp == str(src)
finally:
celery_app.celery.conf.task_always_eager = False
def test_quick_scan_does_not_requeue_completed_task(db_sync, tmp_path):
"""The flip side: quick scan still keeps the old skip semantics —
a file with a completed ImportTask row from a prior batch is NOT
re-enqueued on a fresh quick scan."""
import_root = tmp_path / "import"
settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
settings.import_scan_path = str(import_root)
db_sync.commit()
src = import_root / "Mae" / "q.jpg"
_img(src)
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
first_task = db_sync.execute(
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
).scalar_one()
first_task.status = "complete"
db_sync.commit()
second_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
second_batch_count = db_sync.execute(
select(func.count())
.select_from(ImportTask)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert second_batch_count == 0, (
"quick scan re-queued an already-completed task — should have skipped"
)
# And the second batch should self-finalize (files_seen=0).
second_batch = db_sync.get(ImportBatch, second_batch_id)
assert second_batch.status == "complete"