Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3872e1dda9 | |||
| 17e19081a2 | |||
| 9814f3dbaf | |||
| 770bcf3aa6 | |||
| 52d7905c43 | |||
| e6ededbe8e | |||
| c06cbc0abe | |||
| b214460fdb | |||
| ac39509a74 | |||
| 3531f373ee | |||
| 36cc0622cb | |||
| e50f92d900 | |||
| ba8d9b112d | |||
| c451061ca5 |
@@ -6,18 +6,150 @@ on:
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
# - write:release (for future release-cutting workflows)
|
||||
# - write:release (for ext-<version> release asset cache)
|
||||
# - write:issue (for future issue-management automation)
|
||||
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
||||
|
||||
jobs:
|
||||
build-web:
|
||||
# Sign-or-fetch-from-cache: signs the extension via AMO if no ext-<version>
|
||||
# Forgejo release exists yet, otherwise downloads the cached signed XPI.
|
||||
# Result is uploaded as an Actions artifact for build-web to consume.
|
||||
#
|
||||
# Why this lives in build.yml (not a separate workflow): the merge-commit's
|
||||
# docker image tagged `:latest` MUST carry the XPI. A separate sign workflow
|
||||
# racing build.yml leaves `:latest` without the XPI for ~5min (until the
|
||||
# commit-back triggers another build). Inline ordering eliminates the race.
|
||||
# Cache strategy: Forgejo Release Assets — picked 2026-05-25 over Generic
|
||||
# Packages (cleaner API surface) and commit-back-to-side-branch (no extra
|
||||
# branch to manage). AMO blocks re-signing the same version (returns 409),
|
||||
# so signing is intentionally one-shot per version bump.
|
||||
sign-extension:
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve extension version
|
||||
id: extver
|
||||
run: |
|
||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved extension version: $VERSION"
|
||||
|
||||
- name: Check Forgejo release-asset cache
|
||||
id: cache
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
VERSION=${{ steps.extver.outputs.version }}
|
||||
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||
echo "Tag lookup HTTP status: $STATUS"
|
||||
if [ "$STATUS" = "200" ]; then
|
||||
ASSET_ID=$(jq -r '.assets[]? | select(.name | test("\\.xpi$")) | .id' release.json | head -1)
|
||||
if [ -n "$ASSET_ID" ] && [ "$ASSET_ID" != "null" ]; then
|
||||
echo "cached=true" >> "$GITHUB_OUTPUT"
|
||||
echo "asset_id=$ASSET_ID" >> "$GITHUB_OUTPUT"
|
||||
echo "Cached XPI exists at ext-$VERSION (asset id $ASSET_ID); skipping AMO sign"
|
||||
else
|
||||
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Release ext-$VERSION exists but has no .xpi asset; will re-sign + re-upload"
|
||||
fi
|
||||
else
|
||||
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No release named ext-$VERSION; will sign via AMO and upload"
|
||||
fi
|
||||
|
||||
- name: Download cached signed XPI (cache hit)
|
||||
if: steps.cache.outputs.cached == 'true'
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p extension/web-ext-artifacts
|
||||
curl -sL -H "Authorization: token $TOKEN" \
|
||||
-o "extension/web-ext-artifacts/fabledcurator-${{ steps.extver.outputs.version }}.xpi" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/assets/${{ steps.cache.outputs.asset_id }}"
|
||||
ls -la extension/web-ext-artifacts/
|
||||
|
||||
- name: Sign via AMO (cache miss)
|
||||
if: steps.cache.outputs.cached != 'true'
|
||||
run: |
|
||||
cd extension && npm install --no-save --no-audit --no-fund && npm run sign
|
||||
env:
|
||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||
|
||||
- name: Upload signed XPI to ext-<version> release (cache miss)
|
||||
if: steps.cache.outputs.cached != 'true'
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eux
|
||||
VERSION=${{ steps.extver.outputs.version }}
|
||||
# AMO renames signed XPIs with its internal addon-id-safe-string;
|
||||
# canonicalize to fabledcurator-<version>.xpi so the FC server's
|
||||
# whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
|
||||
# keeps working.
|
||||
SIGNED=$(ls extension/web-ext-artifacts/*.xpi | head -1)
|
||||
XPI="extension/web-ext-artifacts/fabledcurator-$VERSION.xpi"
|
||||
cp "$SIGNED" "$XPI"
|
||||
# Find-or-create the ext-<version> release.
|
||||
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||
if [ "$STATUS" != "200" ]; then
|
||||
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"ext-$VERSION\",\"name\":\"Extension $VERSION (signed XPI cache)\",\"body\":\"Internal cache for the signed XPI consumed by build.yml's build-web job. Not a user-facing FC release.\",\"target_commitish\":\"main\"}" \
|
||||
-o release.json \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases"
|
||||
fi
|
||||
RELEASE_ID=$(jq -r '.id' release.json)
|
||||
test -n "$RELEASE_ID" && test "$RELEASE_ID" != "null"
|
||||
curl -s -X POST -H "Authorization: token $TOKEN" \
|
||||
-F "attachment=@$XPI" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi"
|
||||
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
|
||||
|
||||
- name: Upload XPI as Actions artifact (handoff to build-web)
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: signed-extension
|
||||
path: extension/web-ext-artifacts/fabledcurator-*.xpi
|
||||
retention-days: 1
|
||||
|
||||
build-web:
|
||||
needs: [sign-extension]
|
||||
# sign-extension is main-only; on dev it's skipped, build-web still runs.
|
||||
if: always() && (needs.sign-extension.result == 'success' || needs.sign-extension.result == 'skipped')
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download signed extension (main only)
|
||||
if: github.ref == 'refs/heads/main'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: signed-extension
|
||||
path: extension/web-ext-artifacts/
|
||||
|
||||
- name: Place signed XPI in build context (main only)
|
||||
if: github.ref == 'refs/heads/main'
|
||||
run: |
|
||||
set -eux
|
||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||
XPI=$(ls extension/web-ext-artifacts/*.xpi | head -1)
|
||||
mkdir -p frontend/public/extension
|
||||
cp "$XPI" "frontend/public/extension/fabledcurator-$VERSION.xpi"
|
||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||
ls -la frontend/public/extension/
|
||||
|
||||
- name: Determine tag
|
||||
id: tag
|
||||
run: |
|
||||
|
||||
+168
-22
@@ -72,17 +72,28 @@ jobs:
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
||||
# default bridge with NO service-name DNS, and publishing fixed host
|
||||
# ports collides with the operator's running docker-compose dev stack on
|
||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
||||
# each service by its bridge IP — discovered at runtime via the mounted
|
||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
||||
# values don't depend on cross-step env passing. Pattern documented in
|
||||
# FabledRulebook/forgejo.md "CI philosophy".
|
||||
# Integration suite split into THREE parallel shards (2026-05-25, runner
|
||||
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
|
||||
# set and runs alembic + a disjoint subset of integration tests. Shards
|
||||
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
|
||||
# stays single-threaded per shard but multiple shards run in parallel
|
||||
# wall-clock. Approximate split — rebalance once --durations=15 output
|
||||
# reveals which shard is the long pole.
|
||||
#
|
||||
# Each shard's docker-ps filter uses its own unique job name to scope
|
||||
# service-container resolution. act_runner appears to strip underscores
|
||||
# from job names when building container labels — `int_api` yielded
|
||||
# zero matches on 2026-05-25 — so shards use no-separator names
|
||||
# (`intapi`, `intimp`, `intcore`) instead. Each step prints
|
||||
# `docker ps -a` first so a future naming-convention shift surfaces in
|
||||
# the log without another guess-and-push cycle.
|
||||
#
|
||||
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
|
||||
# done — per ci-requirements.md, FC is the only Python consumer of that
|
||||
# image and the CI-Runner project's "add deps to image when used by >1
|
||||
# project" rule keeps the install per-job.
|
||||
|
||||
intapi:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
@@ -113,7 +124,6 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
@@ -121,15 +131,14 @@ jobs:
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
- name: API integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
# Scope to THIS job's service containers (act_runner names them
|
||||
# ...JOB-integration...); the operator's compose stack uses the
|
||||
# same images but different names, so it won't match.
|
||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -137,16 +146,153 @@ jobs:
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
# uv when available (5-10x faster wheel resolve); fall back to pip.
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration --durations=25
|
||||
pytest tests/test_api_*.py -v -m integration --durations=15
|
||||
|
||||
intimp:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
- name: Importer integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
|
||||
|
||||
intcore:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
|
||||
run: |
|
||||
set -eux
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration --durations=15 \
|
||||
--ignore-glob='tests/test_api_*.py' \
|
||||
--ignore-glob='tests/test_importer*.py' \
|
||||
--ignore-glob='tests/test_import_*.py' \
|
||||
--ignore-glob='tests/test_migration_*.py' \
|
||||
--ignore-glob='tests/test_phash_*.py' \
|
||||
--ignore-glob='tests/test_sidecar_*.py' \
|
||||
--ignore-glob='tests/test_scan_*.py' \
|
||||
--ignore-glob='tests/test_archive_extractor.py' \
|
||||
--ignore-glob='tests/test_backfill_phash.py'
|
||||
|
||||
@@ -1,8 +1,19 @@
|
||||
name: extension
|
||||
# Lint-only workflow. The sign-and-publish dance moved into build.yml's
|
||||
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
||||
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
||||
# eliminating the prior race between build.yml and a separate extension.yml.
|
||||
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
paths: ['extension/**']
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/extension.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
@@ -16,56 +27,3 @@ jobs:
|
||||
run: cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
|
||||
sign-and-publish:
|
||||
needs: lint
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:22-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.RELEASE_TOKEN }}
|
||||
- name: Install web-ext + git
|
||||
run: |
|
||||
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
|
||||
cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Sign XPI
|
||||
run: cd extension && npm run sign
|
||||
env:
|
||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||
- name: Commit signed XPI to frontend/public/extension/
|
||||
run: |
|
||||
set -e
|
||||
# AMO renames signed XPIs using its internal addon-id-safe-string
|
||||
# (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. The
|
||||
# original 'fabledcurator-*' glob never matched and the step
|
||||
# exited 1 even on a successful sign (operator-flagged
|
||||
# 2026-05-25). Match any *.xpi in the artifacts dir — fresh
|
||||
# CI runner means there's exactly one — and rename it on copy
|
||||
# so the FC server's whitelist (backend/app/frontend.py expects
|
||||
# 'fabledcurator-*.xpi') keeps working.
|
||||
XPI=$(ls extension/web-ext-artifacts/*.xpi 2>/dev/null | head -1)
|
||||
if [ -z "$XPI" ]; then
|
||||
echo "No XPI produced by web-ext sign — exiting"
|
||||
ls -la extension/web-ext-artifacts/ || true
|
||||
exit 1
|
||||
fi
|
||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||
DEST="frontend/public/extension/fabledcurator-${VERSION}.xpi"
|
||||
mkdir -p frontend/public/extension
|
||||
# Wipe any prior versions so the directory doesn't grow each release.
|
||||
rm -f frontend/public/extension/fabledcurator-*.xpi
|
||||
cp "$XPI" "$DEST"
|
||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||
git config user.name "FC extension CI"
|
||||
git config user.email "noreply@fabledsword.com"
|
||||
git add frontend/public/extension/
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "ext: publish signed XPI fabledcurator-${VERSION}.xpi"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""import_batch.refreshed counter for deep-scan sidecar re-application
|
||||
|
||||
Revision ID: 0019
|
||||
Revises: 0018
|
||||
Create Date: 2026-05-25
|
||||
|
||||
Adds a `refreshed` counter to `import_batch`, mirroring the existing
|
||||
`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now
|
||||
re-applies sidecar metadata to already-imported files (the IR feature
|
||||
that didn't make the FC port the first time); a "refreshed" outcome
|
||||
increments this counter so the UI can surface "X new, Y refreshed"
|
||||
instead of the misleading "Scan complete — no new files" message.
|
||||
|
||||
server_default=0 backfills existing rows in place — no UPDATE needed.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0019"
|
||||
down_revision: Union[str, None] = "0018"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_batch",
|
||||
sa.Column(
|
||||
"refreshed", sa.Integer(),
|
||||
nullable=False, server_default=sa.text("0"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_batch", "refreshed")
|
||||
@@ -53,6 +53,7 @@ async def status():
|
||||
"imported": active.imported,
|
||||
"skipped": active.skipped,
|
||||
"failed": active.failed,
|
||||
"refreshed": active.refreshed,
|
||||
"started_at": active.started_at.isoformat(),
|
||||
}
|
||||
return jsonify(payload)
|
||||
|
||||
@@ -26,6 +26,10 @@ class ImportBatch(Base):
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||
# running | complete | cancelled
|
||||
|
||||
@@ -51,7 +51,14 @@ class SkipReason(StrEnum):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
||||
# 'imported' — new ImageRecord row created
|
||||
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
|
||||
# 'attached' — non-media saved as PostAttachment
|
||||
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
|
||||
# artist on an already-imported row (no new ImageRecord)
|
||||
# 'skipped' — no work done (true duplicate, too small, etc.)
|
||||
# 'failed' — pipeline error
|
||||
status: str
|
||||
image_id: int | None = None
|
||||
skip_reason: SkipReason | None = None
|
||||
error: str | None = None
|
||||
@@ -71,6 +78,31 @@ def is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
def _safe_ext(path: Path) -> str:
|
||||
"""Conservatively extract a file extension for PostAttachment.ext
|
||||
(varchar(32)).
|
||||
|
||||
gallery-dl produces some filenames with URL-encoded query-string
|
||||
artifacts embedded into the basename (e.g.
|
||||
`79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
|
||||
`Path.suffix` finds the LAST dot and returns everything after, which
|
||||
in those cases yields a 50+ char "extension" of mostly base64-ish
|
||||
junk. That blows the column. Operator-flagged 2026-05-25.
|
||||
|
||||
Real extensions are short and alphanumeric. We accept anything ≤ 16
|
||||
chars where every post-dot character is alphanumeric; anything else
|
||||
means the input wasn't a real extension and we return the empty
|
||||
string. ext is nullable-ish (empty string still satisfies NOT NULL)
|
||||
and consumers should treat "" as "no known extension".
|
||||
"""
|
||||
suffix = path.suffix.lower()
|
||||
if not suffix or len(suffix) > 16:
|
||||
return ""
|
||||
if not all(c.isalnum() for c in suffix[1:]):
|
||||
return ""
|
||||
return suffix
|
||||
|
||||
|
||||
def _mime_for(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
image_mimes = {
|
||||
@@ -126,6 +158,49 @@ class Importer:
|
||||
self.settings = settings
|
||||
self.deep = deep
|
||||
self.attachments = AttachmentStore(images_root)
|
||||
# phash near-dup candidate cache. Archive imports call _import_media
|
||||
# per-member; without this cache the per-member SELECT *FROM
|
||||
# image_record WHERE phash IS NOT NULL fetch repeats N times and a
|
||||
# large library × many-member archive blew past soft_time_limit
|
||||
# (300s) — operator-flagged 2026-05-25. Loaded lazily on first
|
||||
# need, appended to on every imported/superseded outcome, never
|
||||
# invalidated mid-Importer (Importer instances are per-task /
|
||||
# per-archive-import so cross-instance staleness is harmless).
|
||||
self._phash_candidates: list[tuple] | None = None
|
||||
|
||||
def _phash_candidates_cache(self) -> list[tuple]:
|
||||
"""Cached `(phash, width, height, id)` rows from image_record.
|
||||
Loaded on first call, appended-to on subsequent imported/
|
||||
superseded outcomes. Soft-timeout pattern: an archive with N
|
||||
members + a library of M existing rows used to do N × M-row
|
||||
fetches (operator-flagged 2026-05-25); now it's exactly one.
|
||||
|
||||
The per-task lifecycle of Importer (instantiated fresh by
|
||||
import_media_file) bounds the cache's staleness window: cross-
|
||||
process changes (other workers importing concurrently) won't
|
||||
be reflected, but that's the same race the un-cached version
|
||||
had — `find_similar` is best-effort anyway."""
|
||||
if self._phash_candidates is None:
|
||||
rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
self._phash_candidates = [
|
||||
(r.phash, r.width or 0, r.height or 0, r.id) for r in rows
|
||||
]
|
||||
return self._phash_candidates
|
||||
|
||||
def _phash_cache_append(self, phash, width, height, image_id) -> None:
|
||||
"""Append a freshly-imported row to the cache so subsequent
|
||||
members of the same archive can match against it."""
|
||||
if self._phash_candidates is not None and phash is not None:
|
||||
self._phash_candidates.append(
|
||||
(phash, width or 0, height or 0, image_id)
|
||||
)
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||
@@ -209,7 +284,7 @@ class Importer:
|
||||
sha256=sha,
|
||||
path=stored,
|
||||
original_filename=source.name,
|
||||
ext=source.suffix.lower(),
|
||||
ext=_safe_ext(source),
|
||||
mime=_mime_for(source),
|
||||
size_bytes=source.stat().st_size,
|
||||
))
|
||||
@@ -324,18 +399,7 @@ class Importer:
|
||||
error=f"PIL load failed during phash compute: {exc}",
|
||||
)
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
@@ -369,6 +433,7 @@ class Importer:
|
||||
)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Folder→artist (anchored to attribution_path).
|
||||
artist = None
|
||||
@@ -392,13 +457,27 @@ class Importer:
|
||||
) -> ImportResult:
|
||||
"""Deep scan: backfill phash/provenance/artist on an
|
||||
already-imported record. METADATA ONLY — never re-runs the pHash
|
||||
near-dup / supersede path. NULL-only, idempotent."""
|
||||
near-dup / supersede path. NULL-only on phash/artist, additive on
|
||||
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
|
||||
Idempotent: a second deep-scan over the same file finds nothing
|
||||
to refresh and is a no-op.
|
||||
|
||||
Returns status="refreshed" so the UI can surface the work done
|
||||
instead of the prior misleading "skipped/duplicate_hash" reading.
|
||||
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
|
||||
as a no-op skip during the original port and the UI showed deep
|
||||
scan as "completed with no changes" even when sidecar metadata
|
||||
actually got re-applied to N existing rows.
|
||||
"""
|
||||
if existing.phash is None and not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
ph = compute_phash(im)
|
||||
if ph is not None:
|
||||
existing.phash = ph
|
||||
# Promoted from NULL to non-NULL → cache is now stale
|
||||
# (this row would newly qualify for the candidates set).
|
||||
self._phash_candidates = None
|
||||
except Exception as exc:
|
||||
log.warning("deep rephash failed for %s: %s", source, exc)
|
||||
|
||||
@@ -411,10 +490,7 @@ class Importer:
|
||||
|
||||
self._apply_sidecar(existing, attribution_path, artist)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="deep: re-derived",
|
||||
)
|
||||
return ImportResult(status="refreshed", image_id=existing.id)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
@@ -498,18 +574,7 @@ class Importer:
|
||||
except Exception:
|
||||
phash = None
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
@@ -544,6 +609,7 @@ class Importer:
|
||||
record.artist_id = artist.id
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Sidecar provenance (best-effort). When `source` is passed, link
|
||||
# the post to that subscription Source instead of creating a new
|
||||
@@ -759,6 +825,10 @@ class Importer:
|
||||
# created_at intentionally preserved; updated_at auto-bumps.
|
||||
self.session.flush()
|
||||
self.session.commit()
|
||||
# The phash candidate cache (used to avoid N+1 selects during
|
||||
# archive imports) is now stale for `existing.id` — the row's
|
||||
# phash/dimensions changed. Invalidate; the next call re-fetches.
|
||||
self._phash_candidates = None
|
||||
|
||||
# Sidecar enrichment from the new (larger) file's location.
|
||||
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||
|
||||
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
|
||||
def _map_result_to_status(result):
|
||||
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
||||
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb."""
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb.
|
||||
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
|
||||
complete, no ML/thumb re-derive (file/pixels unchanged)."""
|
||||
if result.status in ("imported", "superseded"):
|
||||
return ("complete", True)
|
||||
if result.status == "attached":
|
||||
if result.status in ("attached", "refreshed"):
|
||||
return ("complete", False)
|
||||
if result.status == "skipped":
|
||||
return ("skipped", False)
|
||||
@@ -138,6 +140,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "imported"
|
||||
counter_col = ImportBatch.imported
|
||||
elif result.status == "refreshed":
|
||||
# Deep-scan rederive: existing row got phash/artist/sidecar
|
||||
# refreshed. Task is complete (no further work), but counted in
|
||||
# `refreshed` not `imported` so the UI can surface the actual
|
||||
# work done. operator-flagged 2026-05-25.
|
||||
task.status = "complete"
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "refreshed"
|
||||
counter_col = ImportBatch.refreshed
|
||||
elif result.status == "attached":
|
||||
task.status = "complete"
|
||||
counter_col_name = "attachments"
|
||||
|
||||
@@ -59,16 +59,25 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
session.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
# Skip-set: any source_path that already has a non-failed ImportTask
|
||||
# row. Re-running scan_directory must not re-enqueue files the
|
||||
# importer has already handled (or is currently handling); doing so
|
||||
# creates duplicate work and inflates the queue. Failed prior tasks
|
||||
# are eligible for retry.
|
||||
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
|
||||
#
|
||||
# quick: any non-failed prior ImportTask (active OR finished) is
|
||||
# skipped — quick scan only does new-file enqueue, so re-touching
|
||||
# already-imported files is wasted work.
|
||||
#
|
||||
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
|
||||
# are skipped. Completed and skipped tasks ARE re-queued because
|
||||
# deep scan exists precisely to re-touch already-imported files
|
||||
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
|
||||
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
|
||||
active_statuses = ["pending", "queued", "processing"]
|
||||
if mode == "deep":
|
||||
skip_statuses = active_statuses
|
||||
else:
|
||||
skip_statuses = active_statuses + ["complete", "skipped"]
|
||||
non_failed_existing = set(session.execute(
|
||||
select(ImportTask.source_path).where(
|
||||
ImportTask.status.in_(
|
||||
["pending", "queued", "processing", "complete", "skipped"]
|
||||
),
|
||||
ImportTask.status.in_(skip_statuses),
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.1",
|
||||
"version": "1.0.2",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.1",
|
||||
"version": "1.0.2",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
"lint": "web-ext lint --source-dir=.",
|
||||
"start": "web-ext run --source-dir=. --firefox=firefox",
|
||||
"build": "web-ext build --source-dir=. --overwrite-dest",
|
||||
"sign": "web-ext sign --source-dir=. --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
"lint": "web-ext lint --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore",
|
||||
"start": "web-ext run --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --firefox=firefox",
|
||||
"build": "web-ext build --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --overwrite-dest",
|
||||
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
},
|
||||
"devDependencies": {
|
||||
"web-ext": "^8.0.0"
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
module.exports = {
|
||||
sourceDir: '.',
|
||||
artifactsDir: './web-ext-artifacts',
|
||||
ignoreFiles: [
|
||||
'package.json',
|
||||
'package-lock.json',
|
||||
'web-ext-config.cjs',
|
||||
'web-ext-artifacts',
|
||||
'node_modules',
|
||||
'README.md',
|
||||
'.gitignore',
|
||||
],
|
||||
};
|
||||
@@ -10,6 +10,9 @@
|
||||
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
|
||||
{{ store.activeBatch.source_path || '/import' }} —
|
||||
imported {{ store.activeBatch.imported }},
|
||||
<template v-if="store.activeBatch.scan_mode === 'deep'">
|
||||
refreshed {{ store.activeBatch.refreshed || 0 }},
|
||||
</template>
|
||||
skipped {{ store.activeBatch.skipped }},
|
||||
failed {{ store.activeBatch.failed }} /
|
||||
{{ store.activeBatch.total_files }} files
|
||||
@@ -26,11 +29,12 @@
|
||||
<p class="text-body-2 mb-3">
|
||||
<span v-if="!store.activeBatch">
|
||||
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
|
||||
new files only (skips paths already on a non-failed ImportTask).
|
||||
<strong>Deep scan</strong> additionally chains a phash backfill
|
||||
across the existing library — use after bulk-imports to catch
|
||||
near-duplicates that slipped through. Both modes route non-media
|
||||
+ sidecar pairs through PostAttachment capture.
|
||||
new files only.
|
||||
<strong>Deep scan</strong> additionally re-walks already-imported
|
||||
files so updated sidecar metadata (post title/date/attribution) and
|
||||
previously-NULL phashes / artist links get refreshed. Use after
|
||||
bulk-downloading fresh sidecars for existing content. Both modes
|
||||
route non-media + sidecar pairs through PostAttachment capture.
|
||||
</span>
|
||||
<span v-else>
|
||||
An active batch is in progress. Wait for it to finish, or click
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
<AliasTable class="mt-4" />
|
||||
<BackupCard class="mt-6" />
|
||||
<TagMaintenanceCard class="mt-6" />
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
<LegacyMigrationCard class="mt-6" />
|
||||
</div>
|
||||
</template>
|
||||
@@ -27,7 +26,6 @@ import AllowlistTable from './AllowlistTable.vue'
|
||||
import AliasTable from './AliasTable.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import TagMaintenanceCard from './TagMaintenanceCard.vue'
|
||||
import BrowserExtensionCard from './BrowserExtensionCard.vue'
|
||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
||||
</script>
|
||||
|
||||
|
||||
@@ -66,21 +66,42 @@ export const useImportStore = defineStore('import', () => {
|
||||
// batch flashes 'running' for <100ms then 'complete' before the
|
||||
// first refreshStatus() lands; UI never sees the active state).
|
||||
const label = mode === 'deep'
|
||||
? 'Deep scan triggered (pHash backfill chained)'
|
||||
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
|
||||
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
|
||||
window.__fcToast?.({ text: label, type: 'success' })
|
||||
await refreshStatus()
|
||||
// Re-poll twice over ~5s to catch quick-finalize transitions and
|
||||
// surface a result toast either way. Skip the "no new files" hint
|
||||
// for 'verify' since it doesn't walk the import_root.
|
||||
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
|
||||
// Operator-flagged 2026-05-25: the prior "no new files" message was
|
||||
// misleading because deep scan IS doing work (refresh) even when
|
||||
// there are no new files to import. Surface the real workload count
|
||||
// (imported + refreshed + queued) instead. For quick scan + zero
|
||||
// queued work, fall back to "up to date" instead of the old
|
||||
// implementation-detail-leaking message.
|
||||
setTimeout(async () => {
|
||||
await refreshStatus()
|
||||
await loadTasks(true)
|
||||
if (!activeBatch.value && mode !== 'verify') {
|
||||
if (activeBatch.value || mode === 'verify') return
|
||||
// Batch finalized quickly; figure out what actually happened.
|
||||
// The task list was just refreshed; the freshest row(s) carry
|
||||
// the batch outcome.
|
||||
const batchId = tasks.value[0]?.batch_id
|
||||
const sameBatch = batchId
|
||||
? tasks.value.filter(t => t.batch_id === batchId)
|
||||
: []
|
||||
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
|
||||
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
|
||||
if (mode === 'deep' && sameBatch.length > 0) {
|
||||
window.__fcToast?.({
|
||||
text: 'Scan complete — no new files (everything already on an ImportTask row)',
|
||||
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
|
||||
type: 'info',
|
||||
})
|
||||
} else if (mode === 'quick' && sameBatch.length > 0) {
|
||||
window.__fcToast?.({
|
||||
text: `Quick scan finished — ${newImported} new file(s) queued`,
|
||||
type: 'info',
|
||||
})
|
||||
} else {
|
||||
window.__fcToast?.({ text: 'Library is up to date', type: 'info' })
|
||||
}
|
||||
}, 2000)
|
||||
} catch (e) {
|
||||
|
||||
@@ -45,69 +45,92 @@
|
||||
>Credential health · FC-3b</v-chip>
|
||||
</div>
|
||||
|
||||
<section v-if="store.overview.cooccurring_tags.length" class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Frequent tags</h2>
|
||||
<div class="fc-artist__tags">
|
||||
<v-chip
|
||||
v-for="t in store.overview.cooccurring_tags" :key="t.id"
|
||||
size="small" @click="openTag(t.id)"
|
||||
>{{ t.name }} <span class="fc-artist__tagc">{{ t.count }}</span></v-chip>
|
||||
</div>
|
||||
</section>
|
||||
<!-- Tabs split (2026-05-25): Settings was previously slotted at the
|
||||
bottom of the page after the infinite-scroll image grid, which
|
||||
made it effectively unreachable for any artist with more than
|
||||
a couple of pages of content. The Settings tab now hosts
|
||||
destructive admin actions (artist+content cascade-delete) and
|
||||
any future per-artist management UI. v-tabs is `position:
|
||||
sticky; top: 64px` (under the 64px AppShell TopNav) so it
|
||||
stays parked while the gallery scrolls. -->
|
||||
<v-tabs
|
||||
v-model="tab" color="accent" class="mb-4"
|
||||
style="position: sticky; top: 64px; z-index: 4;
|
||||
background: rgb(var(--v-theme-surface));"
|
||||
>
|
||||
<v-tab value="overview">Overview</v-tab>
|
||||
<v-tab value="settings">Settings</v-tab>
|
||||
</v-tabs>
|
||||
|
||||
<section v-if="store.overview.activity.length" class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Activity</h2>
|
||||
<svg class="fc-artist__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
|
||||
preserveAspectRatio="none" role="img" aria-label="posts over time">
|
||||
<polyline :points="sparkPoints" fill="none"
|
||||
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
|
||||
</svg>
|
||||
</section>
|
||||
<v-window v-model="tab">
|
||||
<v-window-item value="overview">
|
||||
<section v-if="store.overview.cooccurring_tags.length" class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Frequent tags</h2>
|
||||
<div class="fc-artist__tags">
|
||||
<v-chip
|
||||
v-for="t in store.overview.cooccurring_tags" :key="t.id"
|
||||
size="small" @click="openTag(t.id)"
|
||||
>{{ t.name }} <span class="fc-artist__tagc">{{ t.count }}</span></v-chip>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="store.overview.sources.length" class="fc-artist__sec">
|
||||
<div class="fc-artist__sec-head">
|
||||
<h2 class="fc-h2">Sources</h2>
|
||||
<RouterLink
|
||||
:to="`/subscriptions?artist_id=${store.overview.id}`"
|
||||
class="fc-artist__manage"
|
||||
>Manage subscriptions →</RouterLink>
|
||||
</div>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="s in store.overview.sources" :key="s.id">
|
||||
<td>{{ s.platform }}</td>
|
||||
<td class="fc-artist__url">{{ s.url }}</td>
|
||||
<td class="text-right">{{ s.image_count }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</section>
|
||||
<section v-if="store.overview.activity.length" class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Activity</h2>
|
||||
<svg class="fc-artist__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
|
||||
preserveAspectRatio="none" role="img" aria-label="posts over time">
|
||||
<polyline :points="sparkPoints" fill="none"
|
||||
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
|
||||
</svg>
|
||||
</section>
|
||||
|
||||
<section class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Images</h2>
|
||||
<MasonryGrid
|
||||
:items="store.images"
|
||||
:loading="store.imagesLoading"
|
||||
:has-more="store.hasMoreImages"
|
||||
@load-more="store.loadMoreImages(slug)"
|
||||
@open="openImage"
|
||||
/>
|
||||
</section>
|
||||
<section v-if="store.overview.sources.length" class="fc-artist__sec">
|
||||
<div class="fc-artist__sec-head">
|
||||
<h2 class="fc-h2">Sources</h2>
|
||||
<RouterLink
|
||||
:to="`/subscriptions?artist_id=${store.overview.id}`"
|
||||
class="fc-artist__manage"
|
||||
>Manage subscriptions →</RouterLink>
|
||||
</div>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="s in store.overview.sources" :key="s.id">
|
||||
<td>{{ s.platform }}</td>
|
||||
<td class="fc-artist__url">{{ s.url }}</td>
|
||||
<td class="text-right">{{ s.image_count }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</section>
|
||||
|
||||
<ArtistDangerZone
|
||||
:slug="slug"
|
||||
:artist-id="store.overview.id"
|
||||
:artist-name="store.overview.name"
|
||||
/>
|
||||
<section class="fc-artist__sec">
|
||||
<h2 class="fc-h2">Images</h2>
|
||||
<MasonryGrid
|
||||
:items="store.images"
|
||||
:loading="store.imagesLoading"
|
||||
:has-more="store.hasMoreImages"
|
||||
@load-more="store.loadMoreImages(slug)"
|
||||
@open="openImage"
|
||||
/>
|
||||
</section>
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="settings">
|
||||
<ArtistDangerZone
|
||||
:slug="slug"
|
||||
:artist-id="store.overview.id"
|
||||
:artist-name="store.overview.name"
|
||||
/>
|
||||
</v-window-item>
|
||||
</v-window>
|
||||
</template>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, watch } from 'vue'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter, RouterLink } from 'vue-router'
|
||||
import { useArtistStore } from '../stores/artist.js'
|
||||
import { useModalStore } from '../stores/modal.js'
|
||||
@@ -120,8 +143,18 @@ const store = useArtistStore()
|
||||
const modal = useModalStore()
|
||||
|
||||
const slug = computed(() => route.params.slug)
|
||||
// Per-artist tab — defaults to Overview. Settings tab hosts destructive
|
||||
// admin actions (DangerZone). Switching artists resets to Overview so the
|
||||
// destructive surface isn't re-shown by accident when navigating between
|
||||
// artists.
|
||||
const tab = ref('overview')
|
||||
|
||||
watch(slug, (s) => { if (s) store.load(s) }, { immediate: true })
|
||||
watch(slug, (s) => {
|
||||
if (s) {
|
||||
store.load(s)
|
||||
tab.value = 'overview'
|
||||
}
|
||||
}, { immediate: true })
|
||||
|
||||
const dateRange = computed(() => {
|
||||
const r = store.overview?.date_range
|
||||
|
||||
@@ -31,6 +31,11 @@
|
||||
{{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
|
||||
<v-btn variant="text" size="small" @click="tab = 'import'">Go to Import tab</v-btn>
|
||||
</v-alert>
|
||||
<!-- Browser-extension install/download lives on Overview (moved
|
||||
from Maintenance 2026-05-25). Overview is the discovery
|
||||
surface for "things to set up"; Maintenance is for
|
||||
housekeeping of already-set-up systems. -->
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="activity">
|
||||
@@ -62,6 +67,7 @@ import { useImportStore } from '../stores/import.js'
|
||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||
import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
|
||||
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||
|
||||
@@ -59,3 +59,25 @@ def test_json_sidecar_is_not_attached(importer, import_layout):
|
||||
select(func.count()).select_from(PostAttachment)
|
||||
).scalar_one()
|
||||
assert n == 0
|
||||
|
||||
|
||||
def test_mangled_filename_extension_is_sanitized(importer, import_layout):
|
||||
"""gallery-dl sometimes URL-encodes a query string into the basename
|
||||
(`...https___www.patreon.com_media-u_Z0F...`). Python's Path.suffix
|
||||
returns 50+ chars of base64-ish junk for those, which blows the
|
||||
PostAttachment.ext varchar(32) column. Operator-flagged 2026-05-25.
|
||||
The importer should record an empty ext rather than crash."""
|
||||
import_root, _ = import_layout
|
||||
f = (
|
||||
import_root / "Alice"
|
||||
/ "79507046_media_https___www.patreon.com_media-u_Z0FBQUFBQm5q"
|
||||
)
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(b"binary blob")
|
||||
r = importer.import_one(f)
|
||||
assert r.status == "attached"
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
# Junk "extension" -> stored as empty string (not the 50-char garbage).
|
||||
assert att.ext == ""
|
||||
# original_filename is Text-typed so the full name survives intact.
|
||||
assert att.original_filename.endswith("_Z0FBQUFBQm5q")
|
||||
|
||||
@@ -64,9 +64,13 @@ def test_deep_rederives_phash_and_provenance(db_sync, import_layout):
|
||||
|
||||
deep = _mk(db_sync, import_layout, deep=True)
|
||||
r2 = deep.import_one(src)
|
||||
assert r2.status == "skipped"
|
||||
assert "deep" in (r2.error or "")
|
||||
# Outcome flipped from "skipped+duplicate_hash" to "refreshed" 2026-05-25
|
||||
# so the UI can surface deep scan's actual work instead of showing it as
|
||||
# a no-op. See ImportResult.status comment + _deep_rederive docstring.
|
||||
assert r2.status == "refreshed"
|
||||
assert r2.image_id == rec.id
|
||||
assert r2.skip_reason is None
|
||||
assert r2.error is None
|
||||
|
||||
db_sync.expire_all()
|
||||
rec2 = db_sync.get(ImageRecord, rec.id)
|
||||
|
||||
@@ -241,3 +241,9 @@ def test_import_task_maps_superseded_to_complete_and_requeues():
|
||||
assert _map_result_to_status(
|
||||
ImportResult(status="failed", error="boom")
|
||||
) == ("failed", False)
|
||||
# Refreshed (deep scan): complete + no ML/thumb re-derive (pixels
|
||||
# unchanged). Added 2026-05-25 alongside ImportBatch.refreshed
|
||||
# counter so deep scan reports "X refreshed" instead of "no work".
|
||||
assert _map_result_to_status(
|
||||
ImportResult(status="refreshed", image_id=5)
|
||||
) == ("complete", False)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Deep scan re-queues already-completed ImportTasks.
|
||||
|
||||
Operator-flagged 2026-05-25: deep scan used to skip everything that
|
||||
already had a non-failed ImportTask row, making a deep re-scan a no-op
|
||||
when no new files were added. That defeated the entire point of deep
|
||||
scan (re-apply sidecar metadata to existing rows). The skip-set now
|
||||
splits by mode — quick keeps the old "any non-failed" semantics; deep
|
||||
skips ONLY actively-in-flight statuses.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import ImportBatch, ImportSettings, ImportTask
|
||||
from backend.app.tasks.scan import scan_directory
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _img(path):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.new("RGB", (40, 40), (10, 200, 80)).save(path, "JPEG")
|
||||
|
||||
|
||||
def test_deep_scan_requeues_completed_task(db_sync, tmp_path, monkeypatch):
|
||||
"""Quick scan then deep scan of the same /import: the file completed
|
||||
in the first run should be re-enqueued by the deep run (different
|
||||
ImportTask id, same source_path)."""
|
||||
import_root = tmp_path / "import"
|
||||
settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
settings.import_scan_path = str(import_root)
|
||||
db_sync.commit()
|
||||
|
||||
src = import_root / "Mae" / "p.jpg"
|
||||
_img(src)
|
||||
|
||||
from backend.app import celery_app
|
||||
|
||||
celery_app.celery.conf.task_always_eager = False # explicit
|
||||
try:
|
||||
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
|
||||
first_task = db_sync.execute(
|
||||
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
|
||||
).scalar_one()
|
||||
# Simulate the worker having finished it.
|
||||
first_task.status = "complete"
|
||||
db_sync.commit()
|
||||
|
||||
# Now deep scan: the SAME file should get a NEW ImportTask row.
|
||||
second_batch_id = scan_directory.run(triggered_by="manual", mode="deep")
|
||||
assert second_batch_id != first_batch_id
|
||||
|
||||
new_tasks_in_second_batch = db_sync.execute(
|
||||
select(func.count())
|
||||
.select_from(ImportTask)
|
||||
.where(ImportTask.batch_id == second_batch_id)
|
||||
).scalar_one()
|
||||
assert new_tasks_in_second_batch == 1, (
|
||||
"deep scan did not re-queue the completed file"
|
||||
)
|
||||
|
||||
# And the second batch's task should be for the same source_path
|
||||
# as the first (proves it's a re-queue, not a different file).
|
||||
sp = db_sync.execute(
|
||||
select(ImportTask.source_path)
|
||||
.where(ImportTask.batch_id == second_batch_id)
|
||||
).scalar_one()
|
||||
assert sp == str(src)
|
||||
finally:
|
||||
celery_app.celery.conf.task_always_eager = False
|
||||
|
||||
|
||||
def test_quick_scan_does_not_requeue_completed_task(db_sync, tmp_path):
|
||||
"""The flip side: quick scan still keeps the old skip semantics —
|
||||
a file with a completed ImportTask row from a prior batch is NOT
|
||||
re-enqueued on a fresh quick scan."""
|
||||
import_root = tmp_path / "import"
|
||||
settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
settings.import_scan_path = str(import_root)
|
||||
db_sync.commit()
|
||||
|
||||
src = import_root / "Mae" / "q.jpg"
|
||||
_img(src)
|
||||
|
||||
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
|
||||
first_task = db_sync.execute(
|
||||
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
|
||||
).scalar_one()
|
||||
first_task.status = "complete"
|
||||
db_sync.commit()
|
||||
|
||||
second_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
|
||||
second_batch_count = db_sync.execute(
|
||||
select(func.count())
|
||||
.select_from(ImportTask)
|
||||
.where(ImportTask.batch_id == second_batch_id)
|
||||
).scalar_one()
|
||||
assert second_batch_count == 0, (
|
||||
"quick scan re-queued an already-completed task — should have skipped"
|
||||
)
|
||||
|
||||
# And the second batch should self-finalize (files_seen=0).
|
||||
second_batch = db_sync.get(ImportBatch, second_batch_id)
|
||||
assert second_batch.status == "complete"
|
||||
Reference in New Issue
Block a user