Compare commits
126 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 319e8c1d18 | |||
| dcfe55d731 | |||
| e3cdd0f92b | |||
| e77afe8295 | |||
| 57a22d6098 | |||
| a85880f965 | |||
| 407de18ff6 | |||
| b1b129ce9f | |||
| 9075d8eadd | |||
| df6d89cb59 | |||
| 12be188ada | |||
| 6d7116c090 | |||
| b447c42853 | |||
| abafc3265e | |||
| 2394e47370 | |||
| 8243740a04 | |||
| 88e53e5b86 | |||
| aa28bddeab | |||
| b7b313cc05 | |||
| bd3f996582 | |||
| ae8c78ae09 | |||
| 4d2c464045 | |||
| b8ad17c68d | |||
| 1fd54897d8 | |||
| 9322c984fd | |||
| 37e8b796a1 | |||
| 8675f105ad | |||
| 74dac6b960 | |||
| 9e19c081b0 | |||
| 3838f04c16 | |||
| 42b1340324 | |||
| 3b1e2f1ceb | |||
| 8cdf0af0e1 | |||
| ccee344099 | |||
| 0316f92e8b | |||
| 6df74683b3 | |||
| 243e536225 | |||
| 2f16699971 | |||
| a5cb684d34 | |||
| 965a953b2e | |||
| 90c176b195 | |||
| b8d89b9f2a | |||
| 07344e0843 | |||
| 42c33e44f9 | |||
| 4e82208926 | |||
| 85b640f32e | |||
| c7001f4aed | |||
| f827612930 | |||
| 3f0153cba5 | |||
| 52fff00353 | |||
| f3e8f30a8f | |||
| eee107766e | |||
| c14338cbce | |||
| 7a64730bd2 | |||
| 1803a09306 | |||
| 8c36dd28b0 | |||
| 0f7cd3cb76 | |||
| 7b0dd4182c | |||
| 88cfb3dd02 | |||
| fb41b90110 | |||
| 7d84990f6d | |||
| ca55d92c68 | |||
| cce014be3a | |||
| c07effb593 | |||
| a36f72b383 | |||
| 2e8d7c960c | |||
| 992f38ec20 | |||
| 0bc5767a2b | |||
| 397021dcbd | |||
| b0bfbc585a | |||
| 110c1c0e51 | |||
| 6de84d0d60 | |||
| 4e1f208a9f | |||
| 06913eba8e | |||
| b7f693b15e | |||
| ecd0199799 | |||
| 983da9e5b1 | |||
| a41eddae3f | |||
| 7aa7f5a3d6 | |||
| 5d4f223b71 | |||
| 2505b197ae | |||
| 0d0b236ac3 | |||
| a06ada4c9b | |||
| ebd985990c | |||
| 4da8d1d774 | |||
| 05090c6e85 | |||
| 2d4bfa4375 | |||
| 6ed2021ad6 | |||
| 4f2ceaaf31 | |||
| 8a5b337a53 | |||
| 900d878d27 | |||
| fd80d40a34 | |||
| 929d3fc092 | |||
| c0c9e56fb9 | |||
| 3a577d5ade | |||
| 06a2f60c08 | |||
| 0978fbac66 | |||
| f4fe02e346 | |||
| efb142239d | |||
| e766197d99 | |||
| 5587a76606 | |||
| 3872e1dda9 | |||
| 17e19081a2 | |||
| 9814f3dbaf | |||
| 770bcf3aa6 | |||
| 52d7905c43 | |||
| e6ededbe8e | |||
| c06cbc0abe | |||
| b214460fdb | |||
| ac39509a74 | |||
| 3531f373ee | |||
| 36cc0622cb | |||
| e50f92d900 | |||
| ba8d9b112d | |||
| c451061ca5 | |||
| ac55d0e8d8 | |||
| 47d760550d | |||
| 89a89e0ded | |||
| dc3bce7fc1 | |||
| f657582f30 | |||
| 111b952535 | |||
| 4e9aac2c05 | |||
| a0470b5f60 | |||
| b0bb7ae6cc | |||
| 1bbe478fd0 | |||
| 5666fd5ca5 |
@@ -2,26 +2,259 @@ name: Build images
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
# `:dev` builds dropped 2026-05-26 — operator tests from `:latest` after
|
||||
# merge-to-main, not from the dev branch image. Saves one full docker
|
||||
# build per dev push.
|
||||
branches: [main]
|
||||
# Tag-push triggers an immutable per-version image build (e.g.
|
||||
# `:v26.05.26.5`) — gives a real rollback story alongside the floating
|
||||
# `:main` / `:latest`. Layer reuse keeps the registry-storage cost
|
||||
# negligible per tag. Doesn't overlap with the push-to-main build (that
|
||||
# one publishes `:main` + `:latest`; the tag-push build publishes only
|
||||
# `:<tag>`).
|
||||
tags: ['v*']
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
# - write:release (for future release-cutting workflows)
|
||||
# - write:release (for ext-<version> release asset cache)
|
||||
# - write:issue (for future issue-management automation)
|
||||
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
||||
|
||||
jobs:
|
||||
build-web:
|
||||
# Sign-or-fetch-from-cache: signs the extension via AMO if no ext-<version>
|
||||
# Forgejo release exists yet, otherwise downloads the cached signed XPI.
|
||||
# Result is uploaded as an Actions artifact for build-web to consume.
|
||||
#
|
||||
# Why this lives in build.yml (not a separate workflow): the merge-commit's
|
||||
# docker image tagged `:latest` MUST carry the XPI. A separate sign workflow
|
||||
# racing build.yml leaves `:latest` without the XPI for ~5min (until the
|
||||
# commit-back triggers another build). Inline ordering eliminates the race.
|
||||
# Cache strategy: Forgejo Release Assets — picked 2026-05-25 over Generic
|
||||
# Packages (cleaner API surface) and commit-back-to-side-branch (no extra
|
||||
# branch to manage). AMO blocks re-signing the same version (returns 409),
|
||||
# so signing is intentionally one-shot per version bump.
|
||||
sign-extension:
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve extension version
|
||||
id: extver
|
||||
run: |
|
||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved extension version: $VERSION"
|
||||
|
||||
- name: Check Forgejo release-asset cache
|
||||
id: cache
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
VERSION=${{ steps.extver.outputs.version }}
|
||||
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||
echo "Tag lookup HTTP status: $STATUS"
|
||||
# JSON parsing via python (ci-python:3.14 has stdlib json; jq is
|
||||
# not in the image and adding it per ci-requirements.md is not
|
||||
# warranted for a single consumer — operator-flagged 2026-05-26
|
||||
# after a sign job failed with `jq: not found`).
|
||||
if [ "$STATUS" = "200" ]; then
|
||||
ASSET_ID=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['id'] if xpis else '')")
|
||||
if [ -n "$ASSET_ID" ]; then
|
||||
echo "cached=true" >> "$GITHUB_OUTPUT"
|
||||
echo "asset_id=$ASSET_ID" >> "$GITHUB_OUTPUT"
|
||||
echo "Cached XPI exists at ext-$VERSION (asset id $ASSET_ID); skipping AMO sign"
|
||||
else
|
||||
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Release ext-$VERSION exists but has no .xpi asset; will re-sign + re-upload"
|
||||
fi
|
||||
else
|
||||
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No release named ext-$VERSION; will sign via AMO and upload"
|
||||
fi
|
||||
|
||||
# No "download cached XPI in sign-extension" step: build-web
|
||||
# fetches directly from the Forgejo ext-<version> release asset
|
||||
# (removed 2026-05-26 alongside the actions/upload-artifact
|
||||
# removal — sign-extension's job is just to ensure the cache
|
||||
# exists on Forgejo; the build-web side reads it independently).
|
||||
|
||||
- name: Sign via AMO (cache miss)
|
||||
if: steps.cache.outputs.cached != 'true'
|
||||
run: |
|
||||
cd extension && npm install --no-save --no-audit --no-fund && npm run sign
|
||||
env:
|
||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||
|
||||
- name: Upload signed XPI to ext-<version> release (cache miss)
|
||||
if: steps.cache.outputs.cached != 'true'
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eux
|
||||
VERSION=${{ steps.extver.outputs.version }}
|
||||
# AMO renames signed XPIs with its internal addon-id-safe-string;
|
||||
# canonicalize to fabledcurator-<version>.xpi so the FC server's
|
||||
# whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
|
||||
# keeps working.
|
||||
SIGNED=$(ls extension/web-ext-artifacts/*.xpi | head -1)
|
||||
XPI="extension/web-ext-artifacts/fabledcurator-$VERSION.xpi"
|
||||
cp "$SIGNED" "$XPI"
|
||||
# Find-or-create the ext-<version> release. Track whether WE
|
||||
# created it so an upload failure below can roll back (don't
|
||||
# leave an empty release tombstone that the next run's
|
||||
# cache-check mistakes for a partial-failure state).
|
||||
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||
if [ "$STATUS" = "200" ]; then
|
||||
CREATED_BY_US=false
|
||||
else
|
||||
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"ext-$VERSION\",\"name\":\"Extension $VERSION (signed XPI cache)\",\"body\":\"Internal cache for the signed XPI consumed by build.yml's build-web job. Not a user-facing FC release.\",\"target_commitish\":\"main\"}" \
|
||||
-o release.json \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases"
|
||||
CREATED_BY_US=true
|
||||
fi
|
||||
RELEASE_ID=$(python3 -c "import json; print(json.load(open('release.json'))['id'])")
|
||||
test -n "$RELEASE_ID"
|
||||
# Rollback-on-failure: if the asset upload fails AND we just
|
||||
# created the release in this run, delete it. Prevents an empty
|
||||
# ext-<version> release from poisoning the next workflow run
|
||||
# (operator-flagged 2026-05-26 — without rollback the next run
|
||||
# saw 'release exists, no asset → cache miss → sign' which AMO
|
||||
# then rejected with 409 'Version already exists').
|
||||
rollback_if_we_created() {
|
||||
if [ "$CREATED_BY_US" = "true" ]; then
|
||||
echo "Rolling back: deleting just-created release $RELEASE_ID"
|
||||
curl -s -X DELETE -H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID" || true
|
||||
curl -s -X DELETE -H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/tags/ext-$VERSION" || true
|
||||
fi
|
||||
}
|
||||
trap 'rollback_if_we_created' EXIT
|
||||
HTTP_CODE=$(curl -s -X POST -H "Authorization: token $TOKEN" \
|
||||
-F "attachment=@$XPI" \
|
||||
-o /dev/null -w "%{http_code}" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi")
|
||||
if [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "Asset upload failed with HTTP $HTTP_CODE"
|
||||
exit 1
|
||||
fi
|
||||
# Upload succeeded — clear the rollback trap.
|
||||
trap - EXIT
|
||||
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
|
||||
|
||||
# No actions/upload-artifact step: Forgejo Actions (and our
|
||||
# act_runner) doesn't support upload-artifact@v4+ (GHES limitation
|
||||
# surfaced 2026-05-26). Instead build-web reads the signed XPI
|
||||
# straight from the ext-<version> Forgejo release we just uploaded
|
||||
# to. Same source of truth; no double-store.
|
||||
|
||||
build-web:
|
||||
needs: [sign-extension]
|
||||
# sign-extension is main-only; on dev it's skipped, build-web still runs.
|
||||
if: always() && (needs.sign-extension.result == 'success' || needs.sign-extension.result == 'skipped')
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download signed XPI from Forgejo release asset (main + tags)
|
||||
# Fires on main-push AND on tag-push. Tag-push builds re-package the
|
||||
# same source code as the preceding main-push build but with an
|
||||
# immutable version tag — they need the XPI too, otherwise the
|
||||
# versioned image ships without the signed extension.
|
||||
#
|
||||
# Tag-push vs main-push race (operator-flagged 2026-05-27 after
|
||||
# v26.05.27.0 hit it): a release cut fires BOTH workflows almost
|
||||
# simultaneously. Main-push runs sign-extension (1-5min AMO round
|
||||
# trip) before publishing the ext-<version> release; tag-push
|
||||
# skips sign-extension (gated to main) and races straight to
|
||||
# this download step. Tag-push lost every time. Fix: poll the
|
||||
# ext-<version> release endpoint with a sleep+retry loop (30s
|
||||
# for up to 10min total) before giving up. Main-push's signing
|
||||
# eventually wins and tag-push picks the release up on a later
|
||||
# iteration.
|
||||
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
run: |
|
||||
set -eux
|
||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||
# Poll for the ext-<version> release. main-push's sign-extension
|
||||
# step (AMO round-trip, 1-5min) needs to finish + upload before
|
||||
# tag-push can fetch. 30s * 20 = up to 10min wait, then hard-fail.
|
||||
for attempt in $(seq 1 20); do
|
||||
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||
if [ "$STATUS" = "200" ]; then
|
||||
echo "Found ext-$VERSION release on attempt $attempt"
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" = "20" ]; then
|
||||
echo "ERROR: ext-$VERSION release not available after 10min of polling"
|
||||
echo "Last HTTP status: $STATUS"
|
||||
exit 1
|
||||
fi
|
||||
echo "Attempt $attempt: ext-$VERSION not yet published (HTTP $STATUS); sleeping 30s"
|
||||
sleep 30
|
||||
done
|
||||
# Extract the .xpi asset's browser_download_url (Forgejo's
|
||||
# /releases/assets/<id> endpoint returns ASSET METADATA, not
|
||||
# the binary blob — operator-flagged 2026-05-26: my prior
|
||||
# code curl'd the metadata endpoint without -f and wrote the
|
||||
# resulting 404-page-not-found text into fabledcurator-*.xpi,
|
||||
# which Firefox then rejected as "corrupt").
|
||||
# browser_download_url is the canonical binary endpoint and
|
||||
# is also publicly accessible (no token needed) but we pass
|
||||
# the token anyway for symmetry with private-repo support.
|
||||
DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])")
|
||||
test -n "$DOWNLOAD_URL"
|
||||
echo "Downloading XPI from: $DOWNLOAD_URL"
|
||||
mkdir -p frontend/public/extension
|
||||
DEST="frontend/public/extension/fabledcurator-$VERSION.xpi"
|
||||
# -f = fail on HTTP error (prevents silent corruption like the
|
||||
# 2026-05-26 incident); -L = follow redirects.
|
||||
curl -sfL -H "Authorization: token $TOKEN" -o "$DEST" "$DOWNLOAD_URL"
|
||||
# Sanity check: the binary should start with the ZIP magic (PK\x03\x04).
|
||||
# If it's anything else, the next docker build will ship a corrupt XPI.
|
||||
MAGIC=$(head -c 2 "$DEST" | od -An -c | tr -d ' \n')
|
||||
if [ "$MAGIC" != "PK" ]; then
|
||||
echo "ERROR: downloaded XPI does not start with ZIP magic 'PK' (got '$MAGIC')"
|
||||
echo "File contents preview:"
|
||||
head -c 200 "$DEST"
|
||||
exit 1
|
||||
fi
|
||||
cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||
ls -la frontend/public/extension/
|
||||
|
||||
- name: Determine tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
# Three trigger shapes:
|
||||
# refs/tags/v… → tag-push: publish ONLY the immutable version
|
||||
# tag (e.g. :v26.05.26.5). Don't touch :latest;
|
||||
# that already got published by the main-push
|
||||
# build for the merge commit.
|
||||
# refs/heads/main → push to main (incl. PR merge commits):
|
||||
# publish :main + :latest (floating).
|
||||
# anything else → safety net; shouldn't fire given the `on:`
|
||||
# config above (dev was dropped). Tag :dev to
|
||||
# surface the unexpected run in the registry.
|
||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
||||
@@ -52,7 +285,13 @@ jobs:
|
||||
- name: Determine tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
# Mirrors build-web's three-shape logic (tag-push / main-push /
|
||||
# safety-net dev). The -ml image follows the same release cadence
|
||||
# as the web image.
|
||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
||||
|
||||
+183
-24
@@ -8,8 +8,10 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
# pull_request trigger intentionally absent — with branches: [dev, main]
|
||||
# above, every PR commit already fires CI via the push event on dev. Adding
|
||||
# pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
|
||||
# (single-operator Forgejo repo) so push coverage is complete.
|
||||
|
||||
jobs:
|
||||
backend-lint-and-test:
|
||||
@@ -24,11 +26,30 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# Cache step removed 2026-05-26: act_runner's cache backend has been
|
||||
# broken on this homelab runner since 2026-05-15 (first as request-
|
||||
# timeout warnings, then as hard "Cannot find module .../dist/restore/
|
||||
# index.js" failures that tank the whole job). The cache step targeted
|
||||
# ~/.cache/pip but the install below uses `uv pip install` primarily,
|
||||
# whose own cache lives at ~/.cache/uv — so the cache step's real
|
||||
# benefit was marginal even when working. Cost of removal: ~30s of
|
||||
# wheel downloads per job. Future re-enable: mount ~/.cache/uv as a
|
||||
# docker volume at the runner level (skips actions/cache entirely),
|
||||
# or fix the runner-side cache backend (clear /var/run/act/actions/*,
|
||||
# pin act_runner version, etc.).
|
||||
|
||||
- name: Install Python deps
|
||||
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
||||
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
||||
# versions live on the runner image, not here.
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio
|
||||
# uv: 5-10x faster wheel resolve than pip for cold caches.
|
||||
# Falls back to pip install on uv-missing runners (older images).
|
||||
run: |
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
|
||||
- name: Ruff lint
|
||||
run: ruff check backend/ tests/ alembic/
|
||||
@@ -57,17 +78,28 @@ jobs:
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
||||
# default bridge with NO service-name DNS, and publishing fixed host
|
||||
# ports collides with the operator's running docker-compose dev stack on
|
||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
||||
# each service by its bridge IP — discovered at runtime via the mounted
|
||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
||||
# values don't depend on cross-step env passing. Pattern documented in
|
||||
# FabledRulebook/forgejo.md "CI philosophy".
|
||||
# Integration suite split into THREE parallel shards (2026-05-25, runner
|
||||
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
|
||||
# set and runs alembic + a disjoint subset of integration tests. Shards
|
||||
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
|
||||
# stays single-threaded per shard but multiple shards run in parallel
|
||||
# wall-clock. Approximate split — rebalance once --durations=15 output
|
||||
# reveals which shard is the long pole.
|
||||
#
|
||||
# Each shard's docker-ps filter uses its own unique job name to scope
|
||||
# service-container resolution. act_runner appears to strip underscores
|
||||
# from job names when building container labels — `int_api` yielded
|
||||
# zero matches on 2026-05-25 — so shards use no-separator names
|
||||
# (`intapi`, `intimp`, `intcore`) instead. Each step prints
|
||||
# `docker ps -a` first so a future naming-convention shift surfaces in
|
||||
# the log without another guess-and-push cycle.
|
||||
#
|
||||
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
|
||||
# done — per ci-requirements.md, FC is the only Python consumer of that
|
||||
# image and the CI-Runner project's "add deps to image when used by >1
|
||||
# project" rule keeps the install per-job.
|
||||
|
||||
intapi:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
@@ -98,15 +130,14 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
- name: API integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
# Scope to THIS job's service containers (act_runner names them
|
||||
# ...JOB-integration...); the operator's compose stack uses the
|
||||
# same images but different names, so it won't match.
|
||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -114,11 +145,139 @@ jobs:
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration
|
||||
pytest tests/test_api_*.py -v -m integration --durations=15
|
||||
|
||||
intimp:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Importer integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
|
||||
|
||||
intcore:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
|
||||
run: |
|
||||
set -eux
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration --durations=15 \
|
||||
--ignore-glob='tests/test_api_*.py' \
|
||||
--ignore-glob='tests/test_importer*.py' \
|
||||
--ignore-glob='tests/test_import_*.py' \
|
||||
--ignore-glob='tests/test_migration_*.py' \
|
||||
--ignore-glob='tests/test_phash_*.py' \
|
||||
--ignore-glob='tests/test_sidecar_*.py' \
|
||||
--ignore-glob='tests/test_scan_*.py' \
|
||||
--ignore-glob='tests/test_archive_extractor.py' \
|
||||
--ignore-glob='tests/test_backfill_phash.py'
|
||||
|
||||
@@ -1,8 +1,19 @@
|
||||
name: extension
|
||||
# Lint-only workflow. The sign-and-publish dance moved into build.yml's
|
||||
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
||||
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
||||
# eliminating the prior race between build.yml and a separate extension.yml.
|
||||
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
paths: ['extension/**']
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/extension.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
@@ -16,44 +27,3 @@ jobs:
|
||||
run: cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
|
||||
sign-and-publish:
|
||||
needs: lint
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:22-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.RELEASE_TOKEN }}
|
||||
- name: Install web-ext + git
|
||||
run: |
|
||||
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
|
||||
cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Sign XPI
|
||||
run: cd extension && npm run sign
|
||||
env:
|
||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||
- name: Commit signed XPI to frontend/public/extension/
|
||||
run: |
|
||||
set -e
|
||||
XPI=$(ls extension/web-ext-artifacts/fabledcurator-*.xpi | head -1)
|
||||
if [ -z "$XPI" ]; then
|
||||
echo "No XPI produced by web-ext sign — exiting"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p frontend/public/extension
|
||||
cp "$XPI" frontend/public/extension/
|
||||
# Also copy as -latest.xpi so the FC server can serve a stable URL.
|
||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||
git config user.name "FC extension CI"
|
||||
git config user.email "noreply@fabledsword.com"
|
||||
git add frontend/public/extension/
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "ext: publish signed XPI $(basename $XPI)"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""import_batch.refreshed counter for deep-scan sidecar re-application
|
||||
|
||||
Revision ID: 0019
|
||||
Revises: 0018
|
||||
Create Date: 2026-05-25
|
||||
|
||||
Adds a `refreshed` counter to `import_batch`, mirroring the existing
|
||||
`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now
|
||||
re-applies sidecar metadata to already-imported files (the IR feature
|
||||
that didn't make the FC port the first time); a "refreshed" outcome
|
||||
increments this counter so the UI can surface "X new, Y refreshed"
|
||||
instead of the misleading "Scan complete — no new files" message.
|
||||
|
||||
server_default=0 backfills existing rows in place — no UPDATE needed.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0019"
|
||||
down_revision: Union[str, None] = "0018"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_batch",
|
||||
sa.Column(
|
||||
"refreshed", sa.Integer(),
|
||||
nullable=False, server_default=sa.text("0"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_batch", "refreshed")
|
||||
@@ -0,0 +1,65 @@
|
||||
"""fc-cleanup: library_audit_run table for async transparency/single_color audits
|
||||
|
||||
Revision ID: 0020
|
||||
Revises: 0019
|
||||
Create Date: 2026-05-26
|
||||
|
||||
The table backs the async audit lifecycle: rule + params snapshot, status
|
||||
state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and
|
||||
the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs
|
||||
per row by the scan task (oversize = rule too aggressive, operator narrows
|
||||
before re-running).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0020"
|
||||
down_revision: Union[str, None] = "0019"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"library_audit_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("rule", sa.String(32), nullable=False),
|
||||
sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(16),
|
||||
nullable=False, server_default="running",
|
||||
),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"scanned_count", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
sa.Column(
|
||||
"matched_count", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
sa.Column(
|
||||
"matched_ids", postgresql.JSONB(astext_type=sa.Text()),
|
||||
nullable=False, server_default=sa.text("'[]'::jsonb"),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_audit_run_rule", "library_audit_run", ["rule"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_audit_run_status", "library_audit_run", ["status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_library_audit_run_status", table_name="library_audit_run")
|
||||
op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run")
|
||||
op.drop_table("library_audit_run")
|
||||
@@ -0,0 +1,54 @@
|
||||
"""provenance-race: dedupe + UNIQUE(image_record_id, post_id) on image_provenance
|
||||
|
||||
Revision ID: 0021
|
||||
Revises: 0020
|
||||
Create Date: 2026-05-26
|
||||
|
||||
Closes the race in Importer._apply_sidecar's existence-check + INSERT pattern.
|
||||
Two workers writing for the same (image, post) pair both saw no existing row
|
||||
and both inserted, leaving duplicates that then broke .scalar_one_or_none()
|
||||
on every subsequent deep-scan rederive against those images
|
||||
(MultipleResultsFound). Most plausibly seeded when the 5-min recovery sweep
|
||||
re-enqueued a still-running long-import task and the second worker collided
|
||||
with the first inside _apply_sidecar.
|
||||
|
||||
Migration steps:
|
||||
1. DELETE all but min(id) per (image_record_id, post_id) pair. Operator's
|
||||
DB had 2 affected pairs at write-time; harmless no-op if zero.
|
||||
2. Add UNIQUE constraint so the importer's new savepoint+IntegrityError
|
||||
recovery path can trip on collision and re-select, mirroring
|
||||
uq_source_artist_platform_url and uq_post_source_external_id.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0021"
|
||||
down_revision: Union[str, None] = "0020"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM image_provenance ip1
|
||||
USING image_provenance ip2
|
||||
WHERE ip1.image_record_id = ip2.image_record_id
|
||||
AND ip1.post_id = ip2.post_id
|
||||
AND ip1.id > ip2.id
|
||||
"""
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
"uq_image_provenance_image_post",
|
||||
"image_provenance",
|
||||
["image_record_id", "post_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
"uq_image_provenance_image_post",
|
||||
"image_provenance",
|
||||
type_="unique",
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources
|
||||
|
||||
Revision ID: 0022
|
||||
Revises: 0021
|
||||
Create Date: 2026-05-26
|
||||
|
||||
Closes the operator-flagged 2026-05-26 issue where the filesystem importer
|
||||
called _find_or_create_source(url=sd.post_url), creating one Source row per
|
||||
imported post URL. Operator's Atole artist had 406 Source rows where there
|
||||
should have been 1 (the /cw/Atole subscription Source).
|
||||
|
||||
Source represents a subscription feed (one per artist+platform — the
|
||||
gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The
|
||||
filesystem importer was misusing Source as a per-post key.
|
||||
|
||||
Migration steps per (artist_id, platform) group with >1 Source:
|
||||
1. Pick canonical — prefer a URL NOT matching '/posts/<id>$' (real
|
||||
campaign URL like /cw/Atole); else min(id).
|
||||
2. PRE-merge any Posts under non-canonical sources whose
|
||||
external_post_id ALREADY exists under the canonical source. (Same
|
||||
gallery-dl post imported via two different sidecar paths can plant
|
||||
two Post rows with identical external_post_id under different
|
||||
Sources for the same artist.) Repoint ImageProvenance +
|
||||
ImageRecord.primary_post_id to the canonical-side Post, dedupe
|
||||
ImageProvenance against alembic 0021's uq, then delete the
|
||||
non-canonical-side Post. This MUST happen before step 3 — Postgres
|
||||
fires uq_post_source_external_id row-by-row during the bulk UPDATE
|
||||
and the merge-after-reparent ordering 500s on first collision
|
||||
(operator-hit during v26.05.26.1 deploy, 2026-05-26).
|
||||
3. Reparent remaining Posts onto canonical (no collisions possible now).
|
||||
4. Reparent ImageProvenance.source_id off the non-canonical sources.
|
||||
5. Delete the orphan Source rows.
|
||||
6. If the canonical Source's URL still looks like a per-post URL (no
|
||||
campaign URL existed among candidates), rewrite it to
|
||||
'sidecar:<platform>:<artist_slug>' so the artist detail page shows
|
||||
something readable.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0022"
|
||||
down_revision: Union[str, None] = "0021"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_POST_URL_RE = r"/posts/[^/]+$"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Find (artist_id, platform) groups with > 1 Source row.
|
||||
groups = conn.execute(text("""
|
||||
SELECT artist_id, platform
|
||||
FROM source
|
||||
GROUP BY artist_id, platform
|
||||
HAVING COUNT(*) > 1
|
||||
""")).fetchall()
|
||||
|
||||
for artist_id, platform in groups:
|
||||
rows = conn.execute(
|
||||
text("""
|
||||
SELECT id, url FROM source
|
||||
WHERE artist_id = :a AND platform = :p
|
||||
ORDER BY id ASC
|
||||
"""),
|
||||
{"a": artist_id, "p": platform},
|
||||
).fetchall()
|
||||
|
||||
# Canonical: first row whose URL doesn't look like a per-post URL;
|
||||
# else min(id).
|
||||
canonical_id = None
|
||||
for sid, url in rows:
|
||||
if not _matches_post_url(url):
|
||||
canonical_id = sid
|
||||
break
|
||||
if canonical_id is None:
|
||||
canonical_id = rows[0][0]
|
||||
|
||||
other_ids = [sid for sid, _ in rows if sid != canonical_id]
|
||||
if not other_ids:
|
||||
continue
|
||||
|
||||
# STEP 2: PRE-merge ALL Posts with duplicate external_post_id
|
||||
# across the entire (canonical + others) group, BEFORE the bulk
|
||||
# reparent. Two cases must both be handled:
|
||||
# (A) canonical has Post X with epid=N; an "other" source has
|
||||
# Post Y with epid=N → after bulk UPDATE, (canonical, N)
|
||||
# collides with itself.
|
||||
# (B) two different "other" sources each have a Post with
|
||||
# epid=N; canonical has none → after bulk UPDATE, both
|
||||
# are repointed to (canonical, N) and the second collides.
|
||||
# The earlier version of this migration only handled (A); the
|
||||
# operator's deploy 2026-05-26 tripped (B) at line 139.
|
||||
# Fix: group ALL Posts in the (artist, platform) by epid; for
|
||||
# any group with count>1, pick the keep (prefer one already
|
||||
# under canonical; else lowest id) and merge the rest into it.
|
||||
all_posts = conn.execute(
|
||||
text("""
|
||||
SELECT external_post_id, id, source_id
|
||||
FROM post
|
||||
WHERE source_id = :canonical OR source_id = ANY(:others)
|
||||
ORDER BY external_post_id, id
|
||||
"""),
|
||||
{"canonical": canonical_id, "others": other_ids},
|
||||
).fetchall()
|
||||
by_epid: dict = {}
|
||||
for epid, post_id, src_id in all_posts:
|
||||
by_epid.setdefault(epid, []).append((post_id, src_id))
|
||||
for _epid, posts in by_epid.items():
|
||||
if len(posts) <= 1:
|
||||
continue
|
||||
# Prefer a Post already under canonical as the keep.
|
||||
canonical_posts = [p for p in posts if p[1] == canonical_id]
|
||||
if canonical_posts:
|
||||
keep_id = canonical_posts[0][0]
|
||||
else:
|
||||
keep_id = posts[0][0] # already sorted by id ASC
|
||||
drop_ids = [p[0] for p in posts if p[0] != keep_id]
|
||||
for drop_id in drop_ids:
|
||||
# Pre-delete image_provenance rows under drop_ whose
|
||||
# image_record_id ALREADY has a provenance under keep —
|
||||
# the UPDATE below would otherwise repoint them and
|
||||
# trip uq_image_provenance_image_post (alembic 0021)
|
||||
# row-by-row before any after-the-fact dedupe could
|
||||
# run. Operator's v26.05.26.3 deploy 2026-05-26 tripped
|
||||
# this at line 123.
|
||||
conn.execute(
|
||||
text("""
|
||||
DELETE FROM image_provenance
|
||||
WHERE post_id = :drop_
|
||||
AND image_record_id IN (
|
||||
SELECT image_record_id FROM image_provenance
|
||||
WHERE post_id = :keep
|
||||
)
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
# Now safe to repoint the survivors.
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_provenance SET post_id = :keep
|
||||
WHERE post_id = :drop_
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_record SET primary_post_id = :keep
|
||||
WHERE primary_post_id = :drop_
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("DELETE FROM post WHERE id = :drop_"),
|
||||
{"drop_": drop_id},
|
||||
)
|
||||
|
||||
# STEP 3: Bulk reparent the remaining Posts off the other
|
||||
# Sources. After step 2, no collisions on
|
||||
# (canonical, external_post_id) are possible.
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE post SET source_id = :canonical
|
||||
WHERE source_id = ANY(:others)
|
||||
"""),
|
||||
{"canonical": canonical_id, "others": other_ids},
|
||||
)
|
||||
|
||||
# STEP 4: Reparent ImageProvenance.source_id (denormalized FK).
|
||||
# No UNIQUE on source_id; safe bulk update.
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_provenance SET source_id = :canonical
|
||||
WHERE source_id = ANY(:others)
|
||||
"""),
|
||||
{"canonical": canonical_id, "others": other_ids},
|
||||
)
|
||||
|
||||
# STEP 5: Drop the orphan Sources.
|
||||
conn.execute(
|
||||
text("DELETE FROM source WHERE id = ANY(:others)"),
|
||||
{"others": other_ids},
|
||||
)
|
||||
|
||||
# If the canonical's URL still looks per-post (no campaign URL
|
||||
# existed among the candidates), rewrite to a synthetic anchor so
|
||||
# the artist detail page renders something readable.
|
||||
canonical_url = conn.execute(
|
||||
text("SELECT url FROM source WHERE id = :id"),
|
||||
{"id": canonical_id},
|
||||
).scalar_one()
|
||||
if _matches_post_url(canonical_url):
|
||||
slug = conn.execute(
|
||||
text("SELECT slug FROM artist WHERE id = :id"),
|
||||
{"id": artist_id},
|
||||
).scalar_one()
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE source
|
||||
SET url = :new_url, enabled = false
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{
|
||||
"id": canonical_id,
|
||||
"new_url": f"sidecar:{platform}:{slug}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Lossy migration — orphan Sources deleted, Posts reparented, Posts
|
||||
# merged. No safe downgrade. If you need to roll back the schema
|
||||
# invariant, fork from 0021 and re-run filesystem imports.
|
||||
pass
|
||||
|
||||
|
||||
def _matches_post_url(url: str) -> bool:
|
||||
"""True if url ends with /posts/<token> (gallery-dl-style per-post URL)."""
|
||||
import re
|
||||
return bool(re.search(_POST_URL_RE, url or ""))
|
||||
@@ -0,0 +1,99 @@
|
||||
"""drop meta + rating tag kinds — operator-retired 2026-05-26
|
||||
|
||||
Revision ID: 0023
|
||||
Revises: 0022
|
||||
Create Date: 2026-05-26
|
||||
|
||||
Operator decided meta + rating aren't valid tag kinds for FC. Per-row
|
||||
behavior: DELETE existing rows (operator chose "clean break" over
|
||||
"convert to general"). All cascading FKs (image_tag, tag_alias,
|
||||
tag_allowlist, tag_reference_embedding, tag_suggestion_rejection,
|
||||
series_page) use ondelete="CASCADE" so a single DELETE on tag cleans
|
||||
the related rows in one go.
|
||||
|
||||
After the data cleanup, recreate the tag_kind ENUM without 'meta' /
|
||||
'rating' (Postgres has no `ALTER TYPE ... DROP VALUE`; standard
|
||||
rename-create-cast-drop dance). The server default 'general' is
|
||||
dropped before the type swap and restored after.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0023"
|
||||
down_revision: Union[str, None] = "0022"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. Delete tags of the retired kinds. CASCADE handles related tables.
|
||||
op.execute("DELETE FROM tag WHERE kind IN ('meta', 'rating')")
|
||||
|
||||
# 2. Drop the CHECK constraint that references the enum's literal
|
||||
# values. Postgres can't resolve `kind = 'character'` across the
|
||||
# type swap below — the literal would bind to the new tag_kind
|
||||
# but the column is on tag_kind_old, producing
|
||||
# "operator does not exist: tag_kind = tag_kind_old".
|
||||
# (Operator-hit during the v26.05.26.5 deploy attempt; ck was
|
||||
# originally added by alembic 0002.) Recreated post-swap.
|
||||
op.drop_constraint(
|
||||
"ck_tag_fandom_requires_character", "tag", type_="check"
|
||||
)
|
||||
|
||||
# 3. Drop the server default — ALTER COLUMN TYPE can't carry it
|
||||
# across the type swap below.
|
||||
op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
|
||||
|
||||
# 4. Recreate the tag_kind enum without meta/rating.
|
||||
op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
|
||||
op.execute(
|
||||
"CREATE TYPE tag_kind AS ENUM ("
|
||||
"'artist', 'character', 'fandom', 'general', "
|
||||
"'series', 'archive', 'post'"
|
||||
")"
|
||||
)
|
||||
op.execute(
|
||||
"ALTER TABLE tag "
|
||||
"ALTER COLUMN kind TYPE tag_kind "
|
||||
"USING kind::text::tag_kind"
|
||||
)
|
||||
op.execute("DROP TYPE tag_kind_old")
|
||||
|
||||
# 5. Restore the server default.
|
||||
op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
|
||||
|
||||
# 6. Restore the CHECK constraint (now bound to the new tag_kind).
|
||||
op.create_check_constraint(
|
||||
"ck_tag_fandom_requires_character",
|
||||
"tag",
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Add the values back to the enum so old code can boot. The deleted
|
||||
# tag rows are gone permanently — no safe restore.
|
||||
op.drop_constraint(
|
||||
"ck_tag_fandom_requires_character", "tag", type_="check"
|
||||
)
|
||||
op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
|
||||
op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
|
||||
op.execute(
|
||||
"CREATE TYPE tag_kind AS ENUM ("
|
||||
"'artist', 'character', 'fandom', 'general', "
|
||||
"'series', 'archive', 'post', 'meta', 'rating'"
|
||||
")"
|
||||
)
|
||||
op.execute(
|
||||
"ALTER TABLE tag "
|
||||
"ALTER COLUMN kind TYPE tag_kind "
|
||||
"USING kind::text::tag_kind"
|
||||
)
|
||||
op.execute("DROP TYPE tag_kind_old")
|
||||
op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
|
||||
op.create_check_constraint(
|
||||
"ck_tag_fandom_requires_character",
|
||||
"tag",
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""backfill post.post_title from description first-line — 2026-05-27
|
||||
|
||||
Revision ID: 0024
|
||||
Revises: 0023
|
||||
Create Date: 2026-05-27
|
||||
|
||||
SubscribeStar gallery-dl always writes `title: ""` and embeds the leading
|
||||
sentence inside `content` HTML. FC's sidecar parser was leaving
|
||||
post_title NULL for every SubscribeStar post since FC-3 shipped. The
|
||||
parser fix (sidecar._first_line_text fallback) now synthesizes a title
|
||||
at parse time; this migration applies the same logic retroactively to
|
||||
existing rows.
|
||||
|
||||
Operator-flagged 2026-05-27 after inspecting
|
||||
/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars.
|
||||
|
||||
Idempotent: only touches rows where post_title IS NULL or empty AND
|
||||
description IS NOT NULL. Re-running the migration is a no-op.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0024"
|
||||
down_revision: Union[str, None] = "0023"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _first_line_text(body: str, limit: int = 120) -> str | None:
|
||||
"""Mirror of sidecar._first_line_text. Kept inline so the migration
|
||||
doesn't carry a runtime import dependency from app code that may
|
||||
have moved by the time the migration is replayed years from now."""
|
||||
if not body:
|
||||
return None
|
||||
text_ = _TAG_RE.sub(" ", body)
|
||||
text_ = text_.replace("\xa0", " ")
|
||||
for line in text_.splitlines():
|
||||
line = _WS_RE.sub(" ", line).strip()
|
||||
if line:
|
||||
if len(line) > limit:
|
||||
return line[: limit - 1].rstrip() + "…"
|
||||
return line
|
||||
return None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(
|
||||
text(
|
||||
"SELECT id, description FROM post "
|
||||
"WHERE (post_title IS NULL OR post_title = '') "
|
||||
"AND description IS NOT NULL AND description <> ''"
|
||||
)
|
||||
).fetchall()
|
||||
updated = 0
|
||||
for row in rows:
|
||||
derived = _first_line_text(row.description)
|
||||
if not derived:
|
||||
continue
|
||||
bind.execute(
|
||||
text("UPDATE post SET post_title = :t WHERE id = :id"),
|
||||
{"t": derived, "id": row.id},
|
||||
)
|
||||
updated += 1
|
||||
print(f"0024: backfilled post_title on {updated} row(s)")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# No safe restore — we can't tell which post_titles were derived vs
|
||||
# genuinely present. Leave the column alone on rollback.
|
||||
pass
|
||||
@@ -0,0 +1,288 @@
|
||||
"""sidecar-audit followup: correct external_post_id + post_url across all platforms
|
||||
|
||||
Revision ID: 0025
|
||||
Revises: 0024
|
||||
Create Date: 2026-05-27
|
||||
|
||||
Closes the operator-flagged 2026-05-27 sidecar audit findings. Three
|
||||
data-correctness bugs across non-Patreon platforms had been silently
|
||||
corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same
|
||||
commit) addresses new imports. This migration cleans up existing rows.
|
||||
|
||||
Per-platform actions:
|
||||
|
||||
subscribestar — gallery-dl wrote the per-attachment id in `id` and
|
||||
the actual post id in `post_id`. FC's parser picked `id`, so every
|
||||
multi-image SubscribeStar post was fragmented into N Post rows.
|
||||
1. For each SubscribeStar Post, read its sidecar (via the related
|
||||
ImageRecord's on-disk path), pull `post_id`, overwrite
|
||||
external_post_id and post_url.
|
||||
2. Merge groups of Posts under one source that now share an
|
||||
external_post_id (fragments of the same actual post). Same
|
||||
ImageProvenance pre-delete + repoint dance as alembic 0022.
|
||||
|
||||
hentaifoundry — sidecars have NO `url` field; `src` is the image
|
||||
URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar
|
||||
for `user` + `index`, derive the canonical /pictures/user/<u>/<i>
|
||||
permalink. external_post_id (= `index`) was already correct.
|
||||
|
||||
discord — gallery-dl wrote the CDN attachment URL in `url`. FC's
|
||||
parser stored that as post_url. Read each Discord Post's sidecar
|
||||
for the server/channel/message triple, derive the proper
|
||||
discord.com/channels/.../<message> permalink. external_post_id (=
|
||||
`message_id`) was already correct.
|
||||
|
||||
pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on
|
||||
Post.post_url with the derived `/artworks/<id>` permalink. Pixiv
|
||||
external_post_id (= `id`) was already correct; no sidecar IO
|
||||
needed.
|
||||
|
||||
Idempotent: re-running on already-corrected data is a no-op (skips
|
||||
rows whose derived value matches what's already stored).
|
||||
|
||||
Posts whose related ImageRecord paths don't resolve on disk (orphaned
|
||||
filesystem state) are skipped with a count in the migration output —
|
||||
those will be picked up by a future deep-scan.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0025"
|
||||
down_revision: Union[str, None] = "0024"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
# Mirror of sidecar._NUMBERING_PREFIX. Kept inline so the migration is
|
||||
# self-contained (the operator's banked rule:
|
||||
# reference_postgres_enum_swap_drop_checks.md says migrations shouldn't
|
||||
# import from runtime app code).
|
||||
_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
|
||||
|
||||
|
||||
def _find_sidecar(media_path: Path) -> Path | None:
|
||||
"""gallery-dl writes the sidecar under the unprefixed stem
|
||||
(`HOLLOW-ICHIGO.json`) while the media file gets a NN_ ordering
|
||||
prefix (`01_HOLLOW-ICHIGO.png`). Try in order:
|
||||
1. <stem>.json next to the media
|
||||
2. <media>.json next to the media (full-name variant)
|
||||
3. strip the NN_ prefix from the stem, then <stripped>.json
|
||||
"""
|
||||
if not media_path:
|
||||
return None
|
||||
cand = media_path.with_suffix(".json")
|
||||
if cand.is_file():
|
||||
return cand
|
||||
cand = media_path.parent / f"{media_path.name}.json"
|
||||
if cand.is_file():
|
||||
return cand
|
||||
m = _NUMBERING_PREFIX.match(media_path.stem)
|
||||
if m:
|
||||
cand = media_path.parent / f"{m.group(1)}.json"
|
||||
if cand.is_file():
|
||||
return cand
|
||||
return None
|
||||
|
||||
|
||||
def _str_id(v) -> str | None:
|
||||
"""str() a JSON scalar id; reject bool (JSON booleans are ints in
|
||||
Python's eyes but they aren't valid sidecar ids)."""
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
if isinstance(v, (str, int)) and str(v).strip():
|
||||
return str(v).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _str_field(v) -> str | None:
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# ── PART 1: Per-platform corrections requiring filesystem IO ─────
|
||||
# SubscribeStar, HentaiFoundry, Discord all need fields from the
|
||||
# sidecar to construct the right post_url. We walk each Post's
|
||||
# related ImageRecord.path to find the sidecar, read it, derive,
|
||||
# and update.
|
||||
targets = conn.execute(text("""
|
||||
SELECT p.id, p.external_post_id, p.post_url, s.platform
|
||||
FROM post p
|
||||
JOIN source s ON s.id = p.source_id
|
||||
WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord')
|
||||
""")).fetchall()
|
||||
|
||||
stats: dict[str, dict[str, int]] = {
|
||||
plat: {"read": 0, "updated": 0, "no_sidecar": 0}
|
||||
for plat in ("subscribestar", "hentaifoundry", "discord")
|
||||
}
|
||||
for post_row in targets:
|
||||
plat = post_row.platform
|
||||
path = _first_attachment_path(conn, post_row.id)
|
||||
if not path:
|
||||
stats[plat]["no_sidecar"] += 1
|
||||
continue
|
||||
sidecar = _find_sidecar(Path(path))
|
||||
if sidecar is None:
|
||||
stats[plat]["no_sidecar"] += 1
|
||||
continue
|
||||
try:
|
||||
data = json.loads(sidecar.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
stats[plat]["no_sidecar"] += 1
|
||||
continue
|
||||
stats[plat]["read"] += 1
|
||||
|
||||
new_epid = post_row.external_post_id
|
||||
new_url = None
|
||||
if plat == "subscribestar":
|
||||
pid = _str_id(data.get("post_id"))
|
||||
if pid:
|
||||
new_epid = pid
|
||||
new_url = f"https://www.subscribestar.com/posts/{pid}"
|
||||
elif plat == "hentaifoundry":
|
||||
user = _str_field(data.get("user")) or _str_field(data.get("artist"))
|
||||
idx = _str_id(data.get("index"))
|
||||
if user and idx:
|
||||
new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}"
|
||||
elif plat == "discord":
|
||||
sid = _str_id(data.get("server_id"))
|
||||
cid = _str_id(data.get("channel_id"))
|
||||
mid = _str_id(data.get("message_id"))
|
||||
if sid and cid and mid:
|
||||
new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}"
|
||||
|
||||
# Idempotent: skip if nothing changed.
|
||||
if new_epid == post_row.external_post_id and new_url == post_row.post_url:
|
||||
continue
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE post
|
||||
SET external_post_id = :epid, post_url = :url
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"epid": new_epid, "url": new_url, "id": post_row.id},
|
||||
)
|
||||
stats[plat]["updated"] += 1
|
||||
|
||||
for plat, s in stats.items():
|
||||
print(
|
||||
f"0025: {plat} — read {s['read']} sidecars, "
|
||||
f"updated {s['updated']} Posts, "
|
||||
f"{s['no_sidecar']} Posts had no resolvable sidecar"
|
||||
)
|
||||
|
||||
# ── PART 2: Merge SubscribeStar fragments now sharing epid ───────
|
||||
# After Part 1, each group of Posts under one source with the SAME
|
||||
# new external_post_id is a fragment-set of the same actual post.
|
||||
# Merge to one canonical row. Pre-handle the same ImageProvenance
|
||||
# collision pattern as alembic 0022 (uq_image_provenance_image_post).
|
||||
fragment_groups = conn.execute(text("""
|
||||
SELECT p.source_id, p.external_post_id,
|
||||
ARRAY_AGG(p.id ORDER BY p.id ASC) AS post_ids
|
||||
FROM post p
|
||||
JOIN source s ON s.id = p.source_id
|
||||
WHERE s.platform = 'subscribestar'
|
||||
AND p.external_post_id IS NOT NULL
|
||||
GROUP BY p.source_id, p.external_post_id
|
||||
HAVING COUNT(*) > 1
|
||||
""")).fetchall()
|
||||
|
||||
merged = 0
|
||||
for grp in fragment_groups:
|
||||
post_ids = list(grp.post_ids)
|
||||
keep_id, *drop_ids = post_ids
|
||||
for drop_id in drop_ids:
|
||||
# Pre-DELETE colliding ImageProvenance under drop_ that
|
||||
# already exist under keep (alembic 0022 banked the pattern).
|
||||
conn.execute(
|
||||
text("""
|
||||
DELETE FROM image_provenance
|
||||
WHERE post_id = :drop_
|
||||
AND image_record_id IN (
|
||||
SELECT image_record_id FROM image_provenance
|
||||
WHERE post_id = :keep
|
||||
)
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_provenance SET post_id = :keep
|
||||
WHERE post_id = :drop_
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_record SET primary_post_id = :keep
|
||||
WHERE primary_post_id = :drop_
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE post_attachment SET post_id = :keep
|
||||
WHERE post_id = :drop_
|
||||
"""),
|
||||
{"keep": keep_id, "drop_": drop_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("DELETE FROM post WHERE id = :drop_"),
|
||||
{"drop_": drop_id},
|
||||
)
|
||||
merged += 1
|
||||
print(f"0025: subscribestar — merged {merged} duplicate Post fragments")
|
||||
|
||||
# ── PART 3: Pixiv post_url backfill (pure SQL) ───────────────────
|
||||
# Pixiv's external_post_id is already correct (gallery-dl's `id` is
|
||||
# the post id). Only post_url needs derivation: replace anything
|
||||
# under i.pximg.net (the file URL) with the /artworks/<id> permalink.
|
||||
pixiv_updated = conn.execute(text("""
|
||||
UPDATE post p
|
||||
SET post_url = 'https://www.pixiv.net/artworks/' || p.external_post_id
|
||||
FROM source s
|
||||
WHERE p.source_id = s.id
|
||||
AND s.platform = 'pixiv'
|
||||
AND p.external_post_id IS NOT NULL
|
||||
AND (p.post_url IS NULL
|
||||
OR p.post_url LIKE 'https://i.pximg.net/%'
|
||||
OR p.post_url LIKE 'http://i.pximg.net/%')
|
||||
""")).rowcount
|
||||
print(f"0025: pixiv — backfilled post_url on {pixiv_updated} Posts")
|
||||
|
||||
|
||||
def _first_attachment_path(conn, post_id: int) -> str | None:
|
||||
"""Return any ImageRecord.path attached to this post (via
|
||||
ImageProvenance). Lowest-id row keeps the migration deterministic
|
||||
so re-running on the same DB picks the same sidecar."""
|
||||
row = conn.execute(
|
||||
text("""
|
||||
SELECT ir.path
|
||||
FROM image_provenance ip
|
||||
JOIN image_record ir ON ir.id = ip.image_record_id
|
||||
WHERE ip.post_id = :pid
|
||||
ORDER BY ip.id ASC
|
||||
LIMIT 1
|
||||
"""),
|
||||
{"pid": post_id},
|
||||
).first()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Lossy: external_post_id values were overwritten with the correct
|
||||
# post_id; original per-attachment ids weren't preserved. Post-merge
|
||||
# also deleted drop rows. No safe restore. To roll back the schema
|
||||
# invariant, fork from 0024 and re-run sidecar imports.
|
||||
pass
|
||||
@@ -0,0 +1,53 @@
|
||||
"""import_task.recovery_count + refetched — poison-pill circuit breaker
|
||||
|
||||
Revision ID: 0026
|
||||
Revises: 0025
|
||||
Create Date: 2026-05-28
|
||||
|
||||
Backs the import-task resilience work (operator-flagged 2026-05-28):
|
||||
|
||||
- recovery_count: how many times recover_interrupted_tasks has
|
||||
re-queued this row from a stuck 'processing' state. A row that
|
||||
hard-crashes the worker (OOM / segfault on a corrupt or oversized
|
||||
input) leaves no terminal flip, so the sweep re-queues it — and
|
||||
without a cap it would loop forever, re-crashing the worker each
|
||||
time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a
|
||||
diagnostic instead.
|
||||
|
||||
- refetched: whether a one-shot re-download has already been attempted
|
||||
for this task's file. Bounds the Layer-2 re-fetch remediation to a
|
||||
single attempt so source-side corruption doesn't loop.
|
||||
|
||||
Both default to 0 / false; additive, no backfill needed.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0026"
|
||||
down_revision: Union[str, None] = "0025"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_task",
|
||||
sa.Column(
|
||||
"recovery_count", sa.Integer(), nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_task",
|
||||
sa.Column(
|
||||
"refetched", sa.Boolean(), nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_task", "refetched")
|
||||
op.drop_column("import_task", "recovery_count")
|
||||
+38
-1
@@ -3,13 +3,23 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Quart
|
||||
from quart import Quart, request
|
||||
|
||||
from .api import all_blueprints
|
||||
from .config import get_config
|
||||
from .frontend import frontend_bp
|
||||
from .services.credential_crypto import CredentialCrypto
|
||||
|
||||
# Browser-extension origins. The FabledCurator extension fetches from
|
||||
# moz-extension://<uuid>/ on Firefox and chrome-extension://<uuid>/ on
|
||||
# Chromium-based browsers. Operator-flagged 2026-05-26: extension's
|
||||
# 'Test connection' returned `NetworkError` because the X-Extension-Key
|
||||
# header on /api/credentials triggers a CORS preflight that our routes
|
||||
# don't handle. Whitelisting only these two schemes (not opening CORS
|
||||
# up generally) lets the extension talk to a plain-HTTP self-hosted FC
|
||||
# without weakening the no-CORS posture for normal browser usage.
|
||||
_EXTENSION_ORIGIN_SCHEMES = ("moz-extension://", "chrome-extension://")
|
||||
|
||||
_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||
|
||||
|
||||
@@ -35,6 +45,33 @@ def create_app() -> Quart:
|
||||
# Registered last so /api/* routes win over the SPA catch-all.
|
||||
app.register_blueprint(frontend_bp)
|
||||
|
||||
@app.before_request
|
||||
async def _extension_cors_preflight():
|
||||
# Short-circuit OPTIONS preflight from the browser extension with a
|
||||
# 204 + CORS headers (the after_request hook below adds them).
|
||||
# Without this, OPTIONS lands on routes that only declared POST/GET
|
||||
# methods and 405s before the after_request gets a chance.
|
||||
if request.method != "OPTIONS":
|
||||
return None
|
||||
origin = request.headers.get("Origin", "")
|
||||
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||
return "", 204
|
||||
return None
|
||||
|
||||
@app.after_request
|
||||
async def _extension_cors_headers(response):
|
||||
origin = request.headers.get("Origin", "")
|
||||
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||
response.headers["Access-Control-Allow-Origin"] = origin
|
||||
response.headers["Access-Control-Allow-Methods"] = (
|
||||
"GET, POST, PATCH, DELETE, OPTIONS"
|
||||
)
|
||||
response.headers["Access-Control-Allow-Headers"] = (
|
||||
"Content-Type, X-Extension-Key"
|
||||
)
|
||||
response.headers["Access-Control-Max-Age"] = "86400"
|
||||
return response
|
||||
|
||||
@app.after_serving
|
||||
async def _dispose_db_engine() -> None:
|
||||
from .extensions import dispose_engine
|
||||
|
||||
@@ -20,6 +20,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .artist import artist_bp
|
||||
from .artists import artists_bp
|
||||
from .attachments import attachments_bp
|
||||
from .cleanup import cleanup_bp
|
||||
from .credentials import credentials_bp
|
||||
from .downloads import downloads_bp
|
||||
from .extension import extension_bp
|
||||
@@ -37,6 +38,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .system_activity import system_activity_bp
|
||||
from .system_backup import system_backup_bp
|
||||
from .tags import tags_bp
|
||||
from .thumbnails import thumbnails_bp
|
||||
return [
|
||||
api_bp,
|
||||
attachments_bp,
|
||||
@@ -50,12 +52,14 @@ def all_blueprints() -> list[Blueprint]:
|
||||
system_activity_bp,
|
||||
system_backup_bp,
|
||||
admin_bp,
|
||||
cleanup_bp,
|
||||
import_admin_bp,
|
||||
migrate_bp,
|
||||
suggestions_bp,
|
||||
allowlist_bp,
|
||||
aliases_bp,
|
||||
ml_admin_bp,
|
||||
thumbnails_bp,
|
||||
sources_bp,
|
||||
platforms_bp,
|
||||
posts_bp,
|
||||
|
||||
@@ -97,11 +97,19 @@ async def images_bulk_delete():
|
||||
)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return jsonify(projected)
|
||||
|
||||
sha8 = _bulk_image_confirm_token(image_ids)
|
||||
expected = f"delete-images-{sha8}"
|
||||
|
||||
if dry_run:
|
||||
# Hand the canonical Tier-C confirm token back with the
|
||||
# projection so the frontend doesn't have to recompute SHA-256
|
||||
# client-side via crypto.subtle (Secure-Context-gated,
|
||||
# undefined on plain-HTTP origins per the homelab posture).
|
||||
# Operator-flagged 2026-05-27.
|
||||
projected["confirm_token"] = expected
|
||||
return jsonify(projected)
|
||||
|
||||
|
||||
if supplied_confirm != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
"""FC-Cleanup: /api/cleanup/* — retroactive enforcement of import filters.
|
||||
|
||||
Endpoints:
|
||||
POST /min-dimension/preview synchronous SQL audit
|
||||
POST /min-dimension/delete synchronous SQL delete (Tier-C token)
|
||||
POST /audit async transparency / single_color start
|
||||
GET /audit list recent audit_run rows
|
||||
GET /audit/<id> single audit_run row
|
||||
POST /audit/<id>/apply apply matched_ids deletes (Tier-C token)
|
||||
POST /audit/<id>/cancel flip running audit to cancelled
|
||||
|
||||
Unused-tags retroactive prune intentionally NOT in this namespace —
|
||||
TagMaintenanceCard (Maintenance tab → moved to Cleanup tab in v26.05.25.7)
|
||||
uses the existing /api/admin/tags/prune-unused endpoint via the admin
|
||||
store. No duplicate route here.
|
||||
|
||||
Confirm-token format matches modal/DestructiveConfirmModal.vue convention:
|
||||
`delete-min-dim-<sha8(w,h)>` for min-dim delete
|
||||
`delete-audit-<id>` for audit apply
|
||||
(Modal hardcodes action ∈ {'restore', 'delete'}; "apply audit" is semantically a delete of the matched images, so we use `delete-audit-<id>`.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import LibraryAuditRun
|
||||
from ..services import cleanup_service
|
||||
|
||||
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _min_dim_token(min_w: int, min_h: int) -> str:
|
||||
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
||||
# sides use SHA-256 truncated to 8 hex chars.
|
||||
canon = f"{min_w}x{min_h}"
|
||||
return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}"
|
||||
|
||||
|
||||
def _serialize_audit_run(audit: LibraryAuditRun) -> dict:
|
||||
return {
|
||||
"id": audit.id,
|
||||
"rule": audit.rule,
|
||||
"params": audit.params,
|
||||
"status": audit.status,
|
||||
"started_at": audit.started_at.isoformat() if audit.started_at else None,
|
||||
"finished_at": audit.finished_at.isoformat() if audit.finished_at else None,
|
||||
"scanned_count": audit.scanned_count,
|
||||
"matched_count": audit.matched_count,
|
||||
"matched_ids": audit.matched_ids,
|
||||
"error": audit.error,
|
||||
}
|
||||
|
||||
|
||||
@cleanup_bp.route("/min-dimension/preview", methods=["POST"])
|
||||
async def min_dim_preview():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
min_w = int(body.get("min_width", 0))
|
||||
min_h = int(body.get("min_height", 0))
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_dimensions")
|
||||
if min_w < 0 or min_h < 0:
|
||||
return _bad("invalid_dimensions")
|
||||
async with get_session() as session:
|
||||
projection = await session.run_sync(
|
||||
lambda s: cleanup_service.project_min_dimension_violations(
|
||||
s, min_width=min_w, min_height=min_h,
|
||||
)
|
||||
)
|
||||
# Hand the canonical Tier-C delete token back with the preview so
|
||||
# the frontend doesn't have to recompute SHA-256 client-side.
|
||||
# window.crypto.subtle is Secure-Context-gated and undefined on
|
||||
# plain-HTTP origins (homelab posture); without this the Delete
|
||||
# button silently swallowed the TypeError and never opened the
|
||||
# confirm modal. Operator-flagged 2026-05-27.
|
||||
projection["confirm_token"] = _min_dim_token(min_w, min_h)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@cleanup_bp.route("/min-dimension/delete", methods=["POST"])
|
||||
async def min_dim_delete():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
min_w = int(body.get("min_width", 0))
|
||||
min_h = int(body.get("min_height", 0))
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_dimensions")
|
||||
if min_w < 0 or min_h < 0:
|
||||
return _bad("invalid_dimensions")
|
||||
supplied = body.get("confirm", "")
|
||||
expected = _min_dim_token(min_w, min_h)
|
||||
if supplied != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
deleted = await session.run_sync(
|
||||
lambda s: cleanup_service.delete_min_dimension_violations(
|
||||
s, min_width=min_w, min_height=min_h, images_root=IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"deleted": deleted})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit", methods=["POST"])
|
||||
async def audit_create():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
rule = body.get("rule")
|
||||
params = body.get("params") or {}
|
||||
if rule not in ("transparency", "single_color"):
|
||||
return _bad("invalid_rule")
|
||||
if not isinstance(params, dict):
|
||||
return _bad("invalid_params")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
audit_id = await session.run_sync(
|
||||
lambda s: cleanup_service.start_audit_run(
|
||||
s, rule=rule, params=params,
|
||||
)
|
||||
)
|
||||
except cleanup_service.AuditAlreadyRunning as running_id:
|
||||
return _bad(
|
||||
"audit_already_running", status=409,
|
||||
running_id=int(str(running_id)),
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _bad(str(exc))
|
||||
await session.commit()
|
||||
return jsonify({"audit_id": audit_id, "status": "running"}), 202
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>", methods=["GET"])
|
||||
async def audit_get(audit_id: int):
|
||||
async with get_session() as session:
|
||||
audit = (await session.execute(
|
||||
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||
)).scalar_one_or_none()
|
||||
if audit is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_serialize_audit_run(audit))
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit", methods=["GET"])
|
||||
async def audit_history():
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "20")), 100)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LibraryAuditRun)
|
||||
.order_by(LibraryAuditRun.id.desc())
|
||||
.limit(limit)
|
||||
)).scalars().all()
|
||||
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>/apply", methods=["POST"])
|
||||
async def audit_apply(audit_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
confirm = body.get("confirm", "")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
deleted = await session.run_sync(
|
||||
lambda s: cleanup_service.apply_audit_run(
|
||||
s, audit_id=audit_id, confirm_token=confirm,
|
||||
images_root=IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
except cleanup_service.AuditNotReady as exc:
|
||||
return _bad("audit_not_ready", current_status=str(exc))
|
||||
except cleanup_service.ConfirmTokenMismatch as exc:
|
||||
return _bad("confirm_mismatch", expected=str(exc))
|
||||
except ValueError as exc:
|
||||
return _bad("not_found", status=404, detail=str(exc))
|
||||
await session.commit()
|
||||
return jsonify({"deleted": deleted})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>/cancel", methods=["POST"])
|
||||
async def audit_cancel(audit_id: int):
|
||||
async with get_session() as session:
|
||||
await session.run_sync(
|
||||
lambda s: cleanup_service.cancel_audit_run(s, audit_id=audit_id)
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"cancelled": True})
|
||||
@@ -5,8 +5,10 @@ status/source/artist. Returns slim records.
|
||||
Detail view: full DownloadEvent including the metadata JSONB.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
@@ -95,6 +97,35 @@ async def list_downloads():
|
||||
return jsonify([_list_record(e, s, a) for e, s, a in rows])
|
||||
|
||||
|
||||
@downloads_bp.route("/stats", methods=["GET"])
|
||||
async def downloads_stats():
|
||||
"""Status-grouped count over download_event for the dashboard stat chips.
|
||||
|
||||
`?window_hours=` (default 24) bounds by `started_at`. The full set of
|
||||
statuses is always present in the response (zero for missing) so the
|
||||
UI doesn't have to fill in defaults.
|
||||
"""
|
||||
try:
|
||||
window_hours = int(request.args.get("window_hours", "24"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_window_hours"}), 400
|
||||
if window_hours < 1 or window_hours > 24 * 365:
|
||||
return jsonify({"error": "invalid_window_hours"}), 400
|
||||
|
||||
since = datetime.now(UTC) - timedelta(hours=window_hours)
|
||||
out = {"pending": 0, "running": 0, "ok": 0, "error": 0, "skipped": 0}
|
||||
async with get_session() as session:
|
||||
stmt = (
|
||||
select(DownloadEvent.status, func.count())
|
||||
.where(DownloadEvent.started_at >= since)
|
||||
.group_by(DownloadEvent.status)
|
||||
)
|
||||
for status, n in (await session.execute(stmt)).all():
|
||||
if status in out:
|
||||
out[status] = int(n)
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@downloads_bp.route("/<int:event_id>", methods=["GET"])
|
||||
async def get_download(event_id: int):
|
||||
async with get_session() as session:
|
||||
|
||||
@@ -93,10 +93,20 @@ def _read_manifest_sync() -> dict | None:
|
||||
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
||||
if not XPI_DIR.is_dir():
|
||||
return None
|
||||
xpis = sorted(XPI_DIR.glob("fabledcurator-*.xpi"), key=lambda p: p.stat().st_mtime)
|
||||
if not xpis:
|
||||
# Exclude the `fabledcurator-latest.xpi` alias when picking the file to
|
||||
# extract a version from — it's a copy of the latest versioned XPI,
|
||||
# written at the same mtime by build.yml, and would otherwise tie or
|
||||
# win the sort (operator-flagged 2026-05-26: UI displayed "v latest"
|
||||
# because `_extract_version("fabledcurator-latest.xpi")` returns
|
||||
# the literal "latest"). The alias still serves as `latest_url`.
|
||||
versioned = [
|
||||
p for p in XPI_DIR.glob("fabledcurator-*.xpi")
|
||||
if p.name != "fabledcurator-latest.xpi"
|
||||
]
|
||||
if not versioned:
|
||||
return None
|
||||
latest = xpis[-1]
|
||||
versioned.sort(key=lambda p: p.stat().st_mtime)
|
||||
latest = versioned[-1]
|
||||
return {
|
||||
"installed": True,
|
||||
"version": _extract_version(latest.name),
|
||||
|
||||
@@ -47,10 +47,13 @@ async def status():
|
||||
if active:
|
||||
payload["active_batch"] = {
|
||||
"id": active.id,
|
||||
"source_path": active.source_path,
|
||||
"scan_mode": active.scan_mode,
|
||||
"total_files": active.total_files,
|
||||
"imported": active.imported,
|
||||
"skipped": active.skipped,
|
||||
"failed": active.failed,
|
||||
"refreshed": active.refreshed,
|
||||
"started_at": active.started_at.isoformat(),
|
||||
}
|
||||
return jsonify(payload)
|
||||
@@ -100,24 +103,66 @@ async def list_tasks():
|
||||
|
||||
@import_admin_bp.route("/retry-failed", methods=["POST"])
|
||||
async def retry_failed():
|
||||
# Fold SELECT into UPDATE…WHERE…RETURNING — the prior SELECT-then-
|
||||
# UPDATE-WHERE-id-IN pattern blew past psycopg's 65535-parameter
|
||||
# ceiling once failed_ids exceeded ~65k rows.
|
||||
async with get_session() as session:
|
||||
failed_ids = (
|
||||
await session.execute(select(ImportTask.id).where(ImportTask.status == "failed"))
|
||||
).scalars().all()
|
||||
if not failed_ids:
|
||||
return jsonify({"retried": 0})
|
||||
await session.execute(
|
||||
result = await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(failed_ids))
|
||||
.values(status="queued", error=None, started_at=None, finished_at=None)
|
||||
.where(ImportTask.status == "failed")
|
||||
.values(
|
||||
status="queued", error=None,
|
||||
started_at=None, finished_at=None,
|
||||
)
|
||||
.returning(ImportTask.id, ImportTask.task_type)
|
||||
)
|
||||
failed = result.all()
|
||||
if not failed:
|
||||
return jsonify({"retried": 0})
|
||||
await session.commit()
|
||||
|
||||
from ..tasks.import_file import import_media_file
|
||||
for tid in failed_ids:
|
||||
import_media_file.delay(tid)
|
||||
from ..tasks.import_file import enqueue_import
|
||||
for tid, task_type in failed:
|
||||
enqueue_import(tid, task_type)
|
||||
|
||||
return jsonify({"retried": len(failed_ids)})
|
||||
return jsonify({"retried": len(failed)})
|
||||
|
||||
|
||||
@import_admin_bp.route("/tasks/<int:task_id>/refetch", methods=["POST"])
|
||||
async def refetch_task(task_id: int):
|
||||
"""Layer-2 one-shot re-download: delete the (corrupt) file behind a
|
||||
failed import task and re-run its source's downloader to fetch a
|
||||
fresh copy. Only works for files that resolve to an enabled,
|
||||
real-URL subscription Source; filesystem-only imports return
|
||||
no_source.
|
||||
|
||||
Returns one of: refetch_queued (+source_id) / no_source /
|
||||
already_refetched / not_found / not_failed.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(_refetch_task_sync, task_id)
|
||||
if result["status"] == "not_found":
|
||||
return jsonify(result), 404
|
||||
if result["status"] == "not_failed":
|
||||
return jsonify(result), 400
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _refetch_task_sync(session, task_id: int) -> dict:
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import ImportSettings
|
||||
from ..services.refetch_service import attempt_refetch
|
||||
|
||||
task = session.get(ImportTask, task_id)
|
||||
if task is None:
|
||||
return {"status": "not_found"}
|
||||
if task.status != "failed":
|
||||
return {"status": "not_failed"}
|
||||
settings = session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
return attempt_refetch(session, task, Path(settings.import_scan_path))
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||
@@ -135,28 +180,26 @@ async def clear_stuck():
|
||||
autoretry-looped for 2 days after a corrupt-data PIL OSError.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
stuck_ids = (
|
||||
await session.execute(
|
||||
select(ImportTask.id).where(
|
||||
ImportTask.status.in_(["pending", "queued", "processing"])
|
||||
)
|
||||
# Fold SELECT into UPDATE…WHERE — see /retry-failed for the
|
||||
# 65535-parameter ceiling rationale. rowcount is enough here
|
||||
# because we don't need the ids afterward (no .delay()).
|
||||
clear_result = await session.execute(
|
||||
update(ImportTask)
|
||||
.where(
|
||||
ImportTask.status.in_(["pending", "queued", "processing"])
|
||||
)
|
||||
).scalars().all()
|
||||
if stuck_ids:
|
||||
await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(stuck_ids))
|
||||
.values(
|
||||
status="failed",
|
||||
finished_at=datetime.now(UTC),
|
||||
error=(
|
||||
"manually cleared via /api/import/clear-stuck "
|
||||
"— stuck in non-terminal state; retry once "
|
||||
"underlying cause (corrupt file, missing model, "
|
||||
"etc.) is resolved"
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="failed",
|
||||
finished_at=datetime.now(UTC),
|
||||
error=(
|
||||
"manually cleared via /api/import/clear-stuck "
|
||||
"— stuck in non-terminal state; retry once "
|
||||
"underlying cause (corrupt file, missing model, "
|
||||
"etc.) is resolved"
|
||||
),
|
||||
)
|
||||
)
|
||||
tasks_failed = clear_result.rowcount or 0
|
||||
|
||||
# Finalize any 'running' ImportBatch that no longer has any
|
||||
# active children. The "Scanning..." banner is driven by
|
||||
@@ -192,7 +235,7 @@ async def clear_stuck():
|
||||
await session.commit()
|
||||
|
||||
return jsonify({
|
||||
"tasks_failed": len(stuck_ids),
|
||||
"tasks_failed": tasks_failed,
|
||||
"batches_finalized": finalized_batches,
|
||||
})
|
||||
|
||||
|
||||
+32
-5
@@ -15,6 +15,7 @@ from ..services.tag_service import (
|
||||
TagService,
|
||||
TagValidationError,
|
||||
)
|
||||
from ..utils.tag_prefix import parse_kind_prefix
|
||||
|
||||
tags_bp = Blueprint("tags", __name__, url_prefix="/api")
|
||||
|
||||
@@ -105,13 +106,39 @@ async def directory():
|
||||
|
||||
@tags_bp.route("/tags", methods=["POST"])
|
||||
async def create_tag():
|
||||
"""Create a tag. Two input shapes accepted:
|
||||
1. Explicit: {name, kind, fandom_id?} — caller already split, kind wins.
|
||||
2. IR-suffix: {name} where name = "kind:Name" (e.g. "artist:Eric").
|
||||
The server runs parse_kind_prefix(name) to derive kind; the colon
|
||||
and prefix are stripped from the stored tag name. If no recognized
|
||||
prefix is present, the kind defaults to `general`.
|
||||
Explicit kind ALWAYS wins (backward-compat for existing callers).
|
||||
"""
|
||||
body = await request.get_json()
|
||||
if not body or "name" not in body or "kind" not in body:
|
||||
return jsonify({"error": "name and kind required"}), 400
|
||||
if not body or "name" not in body:
|
||||
return jsonify({"error": "name required"}), 400
|
||||
name = body["name"]
|
||||
kind = _coerce_kind(body["kind"])
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400
|
||||
explicit_kind_raw = body.get("kind")
|
||||
|
||||
if explicit_kind_raw is not None:
|
||||
# Caller provided kind — honor it; don't re-parse.
|
||||
kind = _coerce_kind(explicit_kind_raw)
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {explicit_kind_raw!r}"}), 400
|
||||
else:
|
||||
# IR-style: parse "kind:Name" from the raw name.
|
||||
parsed_kind, parsed_name = parse_kind_prefix(name)
|
||||
if parsed_kind is not None:
|
||||
name = parsed_name
|
||||
kind = _coerce_kind(parsed_kind)
|
||||
# parse_kind_prefix only returns kinds from KNOWN_KINDS which
|
||||
# are all valid TagKind members, so _coerce_kind can't return
|
||||
# None here — but defensive.
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {parsed_kind!r}"}), 400
|
||||
else:
|
||||
kind = TagKind.general
|
||||
|
||||
fandom_id = body.get("fandom_id")
|
||||
|
||||
async with get_session() as session:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Thumbnail admin API: backfill trigger."""
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
|
||||
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||
|
||||
|
||||
@thumbnails_bp.route("/backfill", methods=["POST"])
|
||||
async def trigger_backfill():
|
||||
from ..tasks.thumbnail import backfill_thumbnails
|
||||
|
||||
r = backfill_thumbnails.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
@@ -33,6 +33,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.download",
|
||||
"backend.app.tasks.backup",
|
||||
"backend.app.tasks.admin",
|
||||
"backend.app.tasks.library_audit",
|
||||
],
|
||||
)
|
||||
app.conf.update(
|
||||
@@ -47,6 +48,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||
},
|
||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||
task_acks_late=True,
|
||||
|
||||
@@ -11,6 +11,7 @@ from .image_record import ImageRecord
|
||||
from .import_batch import ImportBatch
|
||||
from .import_settings import ImportSettings
|
||||
from .import_task import ImportTask
|
||||
from .library_audit_run import LibraryAuditRun
|
||||
from .migration_run import MigrationRun
|
||||
from .ml_settings import MLSettings
|
||||
from .post import Post
|
||||
@@ -43,6 +44,7 @@ __all__ = [
|
||||
"ImportBatch",
|
||||
"ImportTask",
|
||||
"ImportSettings",
|
||||
"LibraryAuditRun",
|
||||
"MLSettings",
|
||||
"MigrationRun",
|
||||
"TagAlias",
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
"""ImageProvenance — links an ImageRecord to a Post.
|
||||
|
||||
Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate
|
||||
rule (spec §3): when a downloaded image is a pHash dupe of an existing
|
||||
record, we append a new provenance row to the existing record rather than
|
||||
dropping the metadata.
|
||||
One image can have many provenance rows — different posts each contribute
|
||||
metadata (enrich-on-duplicate rule, spec §3: a downloaded image that is a
|
||||
pHash dupe of an existing record gets a NEW provenance row for the new post
|
||||
appended, rather than the metadata being dropped). But the (image, post)
|
||||
pair is unique — alembic 0021 enforces uq_image_provenance_image_post
|
||||
after operator-flagged 2026-05-26 saw _apply_sidecar's existence-check +
|
||||
INSERT race plant duplicates that then broke .scalar_one_or_none() on
|
||||
every later deep-scan rederive (MultipleResultsFound).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -16,6 +20,12 @@ from .base import Base
|
||||
|
||||
class ImageProvenance(Base):
|
||||
__tablename__ = "image_provenance"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"image_record_id", "post_id",
|
||||
name="uq_image_provenance_image_post",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
|
||||
@@ -26,6 +26,10 @@ class ImportBatch(Base):
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||
# running | complete | cancelled
|
||||
|
||||
@@ -8,7 +8,16 @@ been processing longer than the stuck-task threshold.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
@@ -26,6 +35,13 @@ class ImportTask(Base):
|
||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||
|
||||
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
||||
# how many times the stuck-task sweep has re-queued this row; after
|
||||
# the cap it's failed with a diagnostic instead of looping. refetched
|
||||
# bounds the one-shot re-download remediation to a single attempt.
|
||||
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
|
||||
result_image_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""LibraryAuditRun — async transparency / single_color audit lifecycle.
|
||||
|
||||
State machine: running → ready → applied / cancelled / error.
|
||||
matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run
|
||||
reads it and routes through cleanup_service.delete_images.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class LibraryAuditRun(Base):
|
||||
__tablename__ = "library_audit_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
)
|
||||
# running | ready | applied | cancelled | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -35,8 +35,10 @@ class TagKind(StrEnum):
|
||||
series = "series"
|
||||
archive = "archive"
|
||||
post = "post"
|
||||
meta = "meta"
|
||||
rating = "rating"
|
||||
# `meta` and `rating` retired by operator 2026-05-26 (alembic 0023).
|
||||
# `artist` retired in FC-2d-vii-c — artists are first-class entities
|
||||
# via Artist/Source rows now, not tags — but the enum value stays
|
||||
# to keep historic tag rows queryable.
|
||||
|
||||
|
||||
image_tag = Table(
|
||||
|
||||
@@ -111,12 +111,22 @@ class ArtistService:
|
||||
)
|
||||
).all()
|
||||
|
||||
post_count = (
|
||||
await self.session.execute(
|
||||
select(func.count(func.distinct(Post.id)))
|
||||
.select_from(Post)
|
||||
.join(Source, Source.id == Post.source_id)
|
||||
.where(Source.artist_id == aid)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
return {
|
||||
"id": artist.id,
|
||||
"name": artist.name,
|
||||
"slug": artist.slug,
|
||||
"is_subscription": bool(artist.is_subscription),
|
||||
"image_count": int(image_count),
|
||||
"post_count": int(post_count),
|
||||
"date_range": {
|
||||
"min": dmin.isoformat() if dmin else None,
|
||||
"max": dmax.isoformat() if dmax else None,
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool.
|
||||
|
||||
The retroactive library-cleanup tab and (future) import-time filter logic
|
||||
both consume these. Importers should NOT inline rule logic going forward;
|
||||
add the rule here and call from both sides.
|
||||
"""
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Single-color audit: matches images where one color dominates beyond
|
||||
the threshold (within the given Euclidean RGB tolerance). The first
|
||||
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||
future spec can adopt it on the import path too.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
_THUMB_SIZE = (64, 64)
|
||||
|
||||
|
||||
def evaluate(
|
||||
pil_image,
|
||||
*,
|
||||
threshold: float,
|
||||
tolerance: int,
|
||||
) -> bool:
|
||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||
distance) of the dominant color exceeds `threshold`.
|
||||
|
||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||
Alpha channels are stripped; only RGB is considered. Animated images
|
||||
use frame 0 (PIL's default after Image.open without seek).
|
||||
"""
|
||||
im = pil_image
|
||||
if im.mode == "RGBA":
|
||||
im = im.convert("RGB")
|
||||
elif im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
if im.size != _THUMB_SIZE:
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||
pixels = list(im.getdata())
|
||||
if not pixels:
|
||||
return False
|
||||
# Normalize L-mode pixels to RGB tuples for distance math.
|
||||
if isinstance(pixels[0], int):
|
||||
pixels = [(p, p, p) for p in pixels]
|
||||
# Dominant color = mean RGB.
|
||||
n = len(pixels)
|
||||
sum_r = sum(p[0] for p in pixels)
|
||||
sum_g = sum(p[1] for p in pixels)
|
||||
sum_b = sum(p[2] for p in pixels)
|
||||
dom = (sum_r / n, sum_g / n, sum_b / n)
|
||||
tol_sq = tolerance * tolerance
|
||||
within = 0
|
||||
for r, g, b in pixels:
|
||||
dr = r - dom[0]
|
||||
dg = g - dom[1]
|
||||
db = b - dom[2]
|
||||
if dr * dr + dg * dg + db * db <= tol_sq:
|
||||
within += 1
|
||||
return (within / n) > threshold
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Transparency audit: matches images whose transparent-pixel fraction
|
||||
exceeds the threshold. Animated images short-circuit (skipped) to avoid
|
||||
the multi-frame PIL decode that hits Celery's hard time limit."""
|
||||
|
||||
|
||||
def evaluate(pil_image, *, threshold: float) -> bool:
|
||||
"""True iff the image's transparent-pixel fraction exceeds threshold.
|
||||
|
||||
False for non-alpha modes and animated images. Mirrors the import-side
|
||||
Importer._transparency_pct logic so retroactive enforcement matches
|
||||
prospective filtering.
|
||||
"""
|
||||
if getattr(pil_image, "is_animated", False):
|
||||
return False
|
||||
if pil_image.mode not in ("RGBA", "LA") and not (
|
||||
pil_image.mode == "P" and "transparency" in pil_image.info
|
||||
):
|
||||
return False
|
||||
im = pil_image
|
||||
if im.mode != "RGBA":
|
||||
im = im.convert("RGBA")
|
||||
alpha = im.getchannel("A")
|
||||
histogram = alpha.histogram()
|
||||
transparent = histogram[0]
|
||||
total = sum(histogram)
|
||||
pct = transparent / total if total else 0.0
|
||||
return pct > threshold
|
||||
@@ -12,12 +12,14 @@ re-exports from this module and then delete the wrapper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, ImageRecord, Tag
|
||||
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||
from ..models.series_page import SeriesPage
|
||||
from ..models.tag import image_tag
|
||||
|
||||
@@ -365,3 +367,146 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||
)
|
||||
session.commit()
|
||||
return {"deleted": len(ids), "sample_names": sample}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MIN_DIM_SAMPLE_CAP = 50
|
||||
|
||||
|
||||
def project_min_dimension_violations(
|
||||
session: Session, *, min_width: int, min_height: int,
|
||||
) -> dict:
|
||||
"""Return {count, sample_ids} for image_record rows with width or
|
||||
height below the thresholds. Synchronous SQL — no PIL inspection
|
||||
needed since width/height are stored columns."""
|
||||
base = select(ImageRecord.id).where(
|
||||
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||
)
|
||||
count = session.execute(
|
||||
select(func.count()).select_from(base.subquery())
|
||||
).scalar_one()
|
||||
sample_ids = session.execute(
|
||||
base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP)
|
||||
).scalars().all()
|
||||
return {"count": count, "sample_ids": list(sample_ids)}
|
||||
|
||||
|
||||
def delete_min_dimension_violations(
|
||||
session: Session, *, min_width: int, min_height: int, images_root: Path,
|
||||
) -> int:
|
||||
"""Delete every image_record where width<min_w OR height<min_h.
|
||||
Routes through delete_images so file-unlink + cascading FKs
|
||||
(image_tag / image_provenance / etc.) are handled uniformly."""
|
||||
ids = session.execute(
|
||||
select(ImageRecord.id).where(
|
||||
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||
)
|
||||
).scalars().all()
|
||||
if not ids:
|
||||
return 0
|
||||
result = delete_images(
|
||||
session, image_ids=list(ids), images_root=images_root,
|
||||
)
|
||||
return result["images_deleted"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audit lifecycle (transparency + single_color async scans).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AuditAlreadyRunning(Exception):
|
||||
"""Another audit_run is currently in status='running' — wait or
|
||||
cancel it before starting a new one. Surfaces as HTTP 409 in the
|
||||
/api/cleanup/audit POST endpoint."""
|
||||
|
||||
|
||||
class AuditNotReady(Exception):
|
||||
"""apply_audit_run called on an audit whose status is not 'ready'."""
|
||||
|
||||
|
||||
class ConfirmTokenMismatch(Exception):
|
||||
"""Operator-supplied confirm token did not match server-recomputed token."""
|
||||
|
||||
|
||||
_VALID_RULES = ("transparency", "single_color")
|
||||
|
||||
|
||||
def start_audit_run(
|
||||
session: Session, *, rule: str, params: dict[str, Any],
|
||||
) -> int:
|
||||
"""Create a LibraryAuditRun row in status='running' and dispatch the
|
||||
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||
|
||||
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||
has status='running'. Operator must cancel or wait."""
|
||||
if rule not in _VALID_RULES:
|
||||
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||
existing = session.execute(
|
||||
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise AuditAlreadyRunning(existing)
|
||||
audit = LibraryAuditRun(
|
||||
rule=rule,
|
||||
params=params,
|
||||
status="running",
|
||||
scanned_count=0,
|
||||
matched_count=0,
|
||||
matched_ids=[],
|
||||
)
|
||||
session.add(audit)
|
||||
session.flush()
|
||||
audit_id = audit.id
|
||||
# Dispatch after flush so audit_id is populated; commit happens in
|
||||
# the API handler so the audit row + dispatch are visible together.
|
||||
from ..tasks.library_audit import scan_library_for_rule
|
||||
scan_library_for_rule.delay(audit_id)
|
||||
return audit_id
|
||||
|
||||
|
||||
def apply_audit_run(
|
||||
session: Session, *, audit_id: int, confirm_token: str, images_root: Path,
|
||||
) -> int:
|
||||
"""Delete all images in audit_run.matched_ids after confirming token.
|
||||
Marks audit status='applied'. Routes through delete_images so files
|
||||
+ cascading FK rows are handled uniformly."""
|
||||
audit = session.execute(
|
||||
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one_or_none()
|
||||
if audit is None:
|
||||
raise ValueError(f"audit_run {audit_id} not found")
|
||||
if audit.status != "ready":
|
||||
raise AuditNotReady(audit.status)
|
||||
# Token format matches modal/DestructiveConfirmModal.vue convention:
|
||||
# ${action}-${kind}-${runId}. The modal hardcodes action ∈ {'restore',
|
||||
# 'delete'}; "apply audit" is semantically a delete of the matched
|
||||
# images, so we use 'delete-audit-<id>' (not 'apply-audit-<id>').
|
||||
expected = f"delete-audit-{audit_id}"
|
||||
if confirm_token != expected:
|
||||
raise ConfirmTokenMismatch(expected)
|
||||
ids = list(audit.matched_ids or [])
|
||||
deleted = 0
|
||||
if ids:
|
||||
result = delete_images(session, image_ids=ids, images_root=images_root)
|
||||
deleted = result["images_deleted"]
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(status="applied", finished_at=datetime.now(UTC))
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||
"""Flip a running audit_run to 'cancelled'. The scan task checks
|
||||
for status=='cancelled' between batches and exits cleanly."""
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.where(LibraryAuditRun.status == "running")
|
||||
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||
)
|
||||
|
||||
@@ -148,6 +148,7 @@ class CredentialService:
|
||||
return None
|
||||
plaintext = self.crypto.decrypt(row.encrypted_blob)
|
||||
netscape = _to_netscape(plaintext)
|
||||
netscape = _augment_cookies(platform, netscape)
|
||||
self.cookies_dir.mkdir(parents=True, exist_ok=True)
|
||||
out = self.cookies_dir / f"{platform}_cookies.txt"
|
||||
out.write_text(netscape)
|
||||
@@ -163,6 +164,19 @@ class CredentialService:
|
||||
return self.crypto.decrypt(row.encrypted_blob)
|
||||
|
||||
|
||||
def _augment_cookies(platform: str, netscape: str) -> str:
|
||||
"""Delegate to the platform's `augment_cookies` hook if one is
|
||||
registered (subscribestar, hentaifoundry, etc. — see
|
||||
`services/platforms/<name>.py`). No-op when the platform doesn't
|
||||
register a hook (Patreon, DeviantArt). Centralizing the
|
||||
quirks-per-platform in the platforms package means adding a new
|
||||
platform's cookie quirks doesn't require touching this file."""
|
||||
info = PLATFORMS.get(platform)
|
||||
if info is None or info.augment_cookies is None:
|
||||
return netscape
|
||||
return info.augment_cookies(netscape)
|
||||
|
||||
|
||||
def _to_netscape(plaintext: str) -> str:
|
||||
"""Accept either Netscape-format text (the extension's output) or a
|
||||
JSON array of cookie dicts (a manual-paste edge case); produce
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, Source
|
||||
@@ -97,20 +98,40 @@ class ExtensionService:
|
||||
raise UnknownPlatformError(f"no platform pattern matched {url!r}")
|
||||
|
||||
async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]:
|
||||
"""Race-safe find-or-create on Artist by slug. Mirrors the
|
||||
savepoint + IntegrityError recovery pattern used in
|
||||
Importer._find_or_create_source/post (see
|
||||
reference_scalar_one_or_none_duplicates memory). Without this,
|
||||
two concurrent quick-add-source calls hitting the same artist
|
||||
would both miss the existence check and the second INSERT would
|
||||
500 against uq_artist_slug.
|
||||
"""
|
||||
slug = slugify(raw_name)
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
||||
self.session.add(artist)
|
||||
await self.session.flush()
|
||||
return artist, True
|
||||
sp = await self.session.begin_nested()
|
||||
try:
|
||||
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
||||
self.session.add(artist)
|
||||
await self.session.flush()
|
||||
await sp.commit()
|
||||
return artist, True
|
||||
except IntegrityError:
|
||||
await sp.rollback()
|
||||
recovered = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one()
|
||||
return recovered, False
|
||||
|
||||
async def _find_or_create_source(
|
||||
self, *, artist_id: int, platform: str, url: str,
|
||||
) -> tuple[Source, bool]:
|
||||
"""Race-safe — same pattern as _find_or_create_artist above. The
|
||||
uq_source_artist_platform_url constraint catches the duplicate
|
||||
insert; we roll the savepoint back and re-select."""
|
||||
existing = (await self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
@@ -120,8 +141,24 @@ class ExtensionService:
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
src = Source(artist_id=artist_id, platform=platform, url=url, enabled=True)
|
||||
self.session.add(src)
|
||||
await self.session.flush()
|
||||
sp = await self.session.begin_nested()
|
||||
try:
|
||||
src = Source(
|
||||
artist_id=artist_id, platform=platform,
|
||||
url=url, enabled=True,
|
||||
)
|
||||
self.session.add(src)
|
||||
await self.session.flush()
|
||||
await sp.commit()
|
||||
except IntegrityError:
|
||||
await sp.rollback()
|
||||
recovered = (await self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
)).scalar_one()
|
||||
return recovered, False
|
||||
await self.session.commit()
|
||||
return src, True
|
||||
|
||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
@@ -29,6 +30,7 @@ from ..models import (
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from ..utils import safe_probe
|
||||
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
||||
from ..utils.phash import compute_phash, find_similar
|
||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
@@ -51,7 +53,14 @@ class SkipReason(StrEnum):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
||||
# 'imported' — new ImageRecord row created
|
||||
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
|
||||
# 'attached' — non-media saved as PostAttachment
|
||||
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
|
||||
# artist on an already-imported row (no new ImageRecord)
|
||||
# 'skipped' — no work done (true duplicate, too small, etc.)
|
||||
# 'failed' — pipeline error
|
||||
status: str
|
||||
image_id: int | None = None
|
||||
skip_reason: SkipReason | None = None
|
||||
error: str | None = None
|
||||
@@ -71,6 +80,31 @@ def is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
def _safe_ext(path: Path) -> str:
|
||||
"""Conservatively extract a file extension for PostAttachment.ext
|
||||
(varchar(32)).
|
||||
|
||||
gallery-dl produces some filenames with URL-encoded query-string
|
||||
artifacts embedded into the basename (e.g.
|
||||
`79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
|
||||
`Path.suffix` finds the LAST dot and returns everything after, which
|
||||
in those cases yields a 50+ char "extension" of mostly base64-ish
|
||||
junk. That blows the column. Operator-flagged 2026-05-25.
|
||||
|
||||
Real extensions are short and alphanumeric. We accept anything ≤ 16
|
||||
chars where every post-dot character is alphanumeric; anything else
|
||||
means the input wasn't a real extension and we return the empty
|
||||
string. ext is nullable-ish (empty string still satisfies NOT NULL)
|
||||
and consumers should treat "" as "no known extension".
|
||||
"""
|
||||
suffix = path.suffix.lower()
|
||||
if not suffix or len(suffix) > 16:
|
||||
return ""
|
||||
if not all(c.isalnum() for c in suffix[1:]):
|
||||
return ""
|
||||
return suffix
|
||||
|
||||
|
||||
def _mime_for(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
image_mimes = {
|
||||
@@ -126,6 +160,179 @@ class Importer:
|
||||
self.settings = settings
|
||||
self.deep = deep
|
||||
self.attachments = AttachmentStore(images_root)
|
||||
# phash near-dup candidate cache. Archive imports call _import_media
|
||||
# per-member; without this cache the per-member SELECT *FROM
|
||||
# image_record WHERE phash IS NOT NULL fetch repeats N times and a
|
||||
# large library × many-member archive blew past soft_time_limit
|
||||
# (300s) — operator-flagged 2026-05-25. Loaded lazily on first
|
||||
# need, appended to on every imported/superseded outcome, never
|
||||
# invalidated mid-Importer (Importer instances are per-task /
|
||||
# per-archive-import so cross-instance staleness is harmless).
|
||||
self._phash_candidates: list[tuple] | None = None
|
||||
|
||||
def _phash_candidates_cache(self) -> list[tuple]:
|
||||
"""Cached `(phash, width, height, id)` rows from image_record.
|
||||
Loaded on first call, appended-to on subsequent imported/
|
||||
superseded outcomes. Soft-timeout pattern: an archive with N
|
||||
members + a library of M existing rows used to do N × M-row
|
||||
fetches (operator-flagged 2026-05-25); now it's exactly one.
|
||||
|
||||
The per-task lifecycle of Importer (instantiated fresh by
|
||||
import_media_file) bounds the cache's staleness window: cross-
|
||||
process changes (other workers importing concurrently) won't
|
||||
be reflected, but that's the same race the un-cached version
|
||||
had — `find_similar` is best-effort anyway."""
|
||||
if self._phash_candidates is None:
|
||||
rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
self._phash_candidates = [
|
||||
(r.phash, r.width or 0, r.height or 0, r.id) for r in rows
|
||||
]
|
||||
return self._phash_candidates
|
||||
|
||||
def _phash_cache_append(self, phash, width, height, image_id) -> None:
|
||||
"""Append a freshly-imported row to the cache so subsequent
|
||||
members of the same archive can match against it."""
|
||||
if self._phash_candidates is not None and phash is not None:
|
||||
self._phash_candidates.append(
|
||||
(phash, width or 0, height or 0, image_id)
|
||||
)
|
||||
|
||||
def _find_or_create_source(
|
||||
self, *, artist_id: int, platform: str, url: str,
|
||||
) -> Source:
|
||||
"""Race-safe find-or-create on `source` keyed by
|
||||
(artist_id, platform, url) — the same key as the
|
||||
`uq_source_artist_platform_url` constraint.
|
||||
|
||||
Two concurrent workers processing different files in the same
|
||||
post can both find no existing Source row then both INSERT,
|
||||
which trips the unique constraint and poisons the session with
|
||||
`psycopg.errors.UniqueViolation`. Operator-flagged 2026-05-26.
|
||||
|
||||
Pattern: select; if absent, open a savepoint and INSERT.
|
||||
On IntegrityError, roll the savepoint back (NOT the outer
|
||||
transaction, which would lose the surrounding scan's progress)
|
||||
and re-select — the concurrent op just created the row we
|
||||
wanted, so the second select will find it.
|
||||
"""
|
||||
existing = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
row = Source(artist_id=artist_id, platform=platform, url=url)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return row
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
return self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
def _source_for_sidecar(
|
||||
self, *, artist_id: int, platform: str, artist_slug: str,
|
||||
) -> Source:
|
||||
"""Filesystem-import sidecar Source resolver.
|
||||
|
||||
Source represents a subscription feed (one per artist+platform — the
|
||||
gallery-dl URL polled by the FC-3 downloader). The filesystem importer
|
||||
used to call _find_or_create_source(url=sd.post_url), which created
|
||||
one Source row per post URL — 100s of junk Sources per artist, all
|
||||
with enabled=True, polluting the artist detail page and tricking the
|
||||
subscription checker into trying to poll patreon post URLs as feeds.
|
||||
Operator-flagged 2026-05-26.
|
||||
|
||||
New behaviour: if any Source row exists for (artist_id, platform),
|
||||
reuse it regardless of its URL — the artist's real subscription Source
|
||||
(created by the downloader / extension / UI) is the canonical
|
||||
attachment point for filesystem-imported posts. If none exists, create
|
||||
ONE synthetic anchor with url='sidecar:<platform>:<artist_slug>' and
|
||||
enabled=False (so the subscription checker doesn't poll it).
|
||||
"""
|
||||
existing = self.session.execute(
|
||||
select(Source)
|
||||
.where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
synthetic_url = f"sidecar:{platform}:{artist_slug}"
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
row = Source(
|
||||
artist_id=artist_id,
|
||||
platform=platform,
|
||||
url=synthetic_url,
|
||||
enabled=False,
|
||||
)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return row
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
return self.session.execute(
|
||||
select(Source)
|
||||
.where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
.limit(1)
|
||||
).scalar_one()
|
||||
|
||||
def _find_or_create_post(
|
||||
self, *, source_id: int, external_post_id: str,
|
||||
) -> Post:
|
||||
"""Race-safe find-or-create on `post` keyed by
|
||||
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||
— same savepoint + IntegrityError-recovery pattern."""
|
||||
existing = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
row = Post(source_id=source_id, external_post_id=external_post_id)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return row
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
return self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||
@@ -165,30 +372,13 @@ class Importer:
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
src = self._source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform, artist_slug=artist.slug,
|
||||
)
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
return post
|
||||
return self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
)
|
||||
|
||||
def _capture_attachment(
|
||||
self, source: Path, *, post: Post | None = None,
|
||||
@@ -209,7 +399,7 @@ class Importer:
|
||||
sha256=sha,
|
||||
path=stored,
|
||||
original_filename=source.name,
|
||||
ext=source.suffix.lower(),
|
||||
ext=_safe_ext(source),
|
||||
mime=_mime_for(source),
|
||||
size_bytes=source.stat().st_size,
|
||||
))
|
||||
@@ -218,6 +408,29 @@ class Importer:
|
||||
return ImportResult(status="attached")
|
||||
|
||||
def _import_archive(self, source: Path) -> ImportResult:
|
||||
# Layer-3 isolation: bomb-size guard + integrity test in a
|
||||
# spawned child BEFORE extracting in this process. A
|
||||
# decompression bomb or a native-lib crash on a malformed
|
||||
# archive is contained to the child; we reject the file cleanly
|
||||
# instead of OOMing/segfaulting the import worker. extract_archive
|
||||
# is already fail-soft for plain exceptions, so this only adds
|
||||
# the hard-crash protection.
|
||||
probe = safe_probe.probe_archive(source)
|
||||
if not probe.ok:
|
||||
if probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
# Clean rejection (bomb cap exceeded, integrity mismatch):
|
||||
# still preserve the archive file itself as an attachment so
|
||||
# nothing silently vanishes, matching extract_archive's
|
||||
# fail-soft contract.
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
self._capture_attachment(source, post=post, artist=artist, resolved=True)
|
||||
return ImportResult(status="attached")
|
||||
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
member_ids: list[int] = []
|
||||
@@ -257,7 +470,25 @@ class Importer:
|
||||
# Compute file dimensions (images only) and apply filters.
|
||||
width = height = None
|
||||
has_alpha = False
|
||||
if not is_video(source):
|
||||
if is_video(source):
|
||||
# Layer-3 isolation: validate the container via ffprobe (a
|
||||
# separate process) before the rest of the pipeline touches
|
||||
# it. A corrupt video that would crash a decoder is rejected
|
||||
# cleanly here, and we capture width/height for free (the
|
||||
# importer didn't previously record video dimensions).
|
||||
probe = safe_probe.probe_video(source)
|
||||
if not probe.ok:
|
||||
if probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"video probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=probe.reason,
|
||||
)
|
||||
width, height = probe.width, probe.height
|
||||
else:
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
im.verify()
|
||||
@@ -324,18 +555,7 @@ class Importer:
|
||||
error=f"PIL load failed during phash compute: {exc}",
|
||||
)
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
@@ -369,6 +589,7 @@ class Importer:
|
||||
)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Folder→artist (anchored to attribution_path).
|
||||
artist = None
|
||||
@@ -392,13 +613,27 @@ class Importer:
|
||||
) -> ImportResult:
|
||||
"""Deep scan: backfill phash/provenance/artist on an
|
||||
already-imported record. METADATA ONLY — never re-runs the pHash
|
||||
near-dup / supersede path. NULL-only, idempotent."""
|
||||
near-dup / supersede path. NULL-only on phash/artist, additive on
|
||||
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
|
||||
Idempotent: a second deep-scan over the same file finds nothing
|
||||
to refresh and is a no-op.
|
||||
|
||||
Returns status="refreshed" so the UI can surface the work done
|
||||
instead of the prior misleading "skipped/duplicate_hash" reading.
|
||||
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
|
||||
as a no-op skip during the original port and the UI showed deep
|
||||
scan as "completed with no changes" even when sidecar metadata
|
||||
actually got re-applied to N existing rows.
|
||||
"""
|
||||
if existing.phash is None and not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
ph = compute_phash(im)
|
||||
if ph is not None:
|
||||
existing.phash = ph
|
||||
# Promoted from NULL to non-NULL → cache is now stale
|
||||
# (this row would newly qualify for the candidates set).
|
||||
self._phash_candidates = None
|
||||
except Exception as exc:
|
||||
log.warning("deep rephash failed for %s: %s", source, exc)
|
||||
|
||||
@@ -411,10 +646,7 @@ class Importer:
|
||||
|
||||
self._apply_sidecar(existing, attribution_path, artist)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="deep: re-derived",
|
||||
)
|
||||
return ImportResult(status="refreshed", image_id=existing.id)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
@@ -498,18 +730,7 @@ class Importer:
|
||||
except Exception:
|
||||
phash = None
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
@@ -544,6 +765,7 @@ class Importer:
|
||||
record.artist_id = artist.id
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Sidecar provenance (best-effort). When `source` is passed, link
|
||||
# the post to that subscription Source instead of creating a new
|
||||
@@ -638,30 +860,15 @@ class Importer:
|
||||
src = explicit_source
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
src = self._source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
artist_slug=artist.slug,
|
||||
)
|
||||
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
post = self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
)
|
||||
if sd.post_url is not None:
|
||||
post.post_url = sd.post_url
|
||||
if sd.post_title is not None:
|
||||
@@ -674,6 +881,15 @@ class Importer:
|
||||
post.attachment_count = sd.attachment_count
|
||||
post.raw_metadata = sd.raw
|
||||
|
||||
# Race-safe (image_record_id, post_id) upsert — mirrors the
|
||||
# _find_or_create_source/post savepoint pattern. The plain
|
||||
# SELECT-then-INSERT pattern lost a race when two workers ran
|
||||
# _apply_sidecar on the same (image, post) pair (e.g. the 5-min
|
||||
# recovery sweep re-enqueued a still-running long import), planting
|
||||
# duplicates that then broke .scalar_one_or_none() on every later
|
||||
# deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
|
||||
# uq_image_provenance_image_post UNIQUE so this savepoint actually
|
||||
# trips on collision.
|
||||
exists = self.session.execute(
|
||||
select(ImageProvenance.id).where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
@@ -681,14 +897,20 @@ class Importer:
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists is None:
|
||||
self.session.add(
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=src.id,
|
||||
captured_metadata=sd.raw,
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
self.session.add(
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=src.id,
|
||||
captured_metadata=sd.raw,
|
||||
)
|
||||
)
|
||||
)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
if record.primary_post_id is None:
|
||||
record.primary_post_id = post.id
|
||||
self.session.flush()
|
||||
@@ -722,6 +944,15 @@ class Importer:
|
||||
row id (so tags/series/curation stay attached). ML is cleared so
|
||||
the import task re-derives it on the new pixels.
|
||||
|
||||
After the file swap, the new file's adjacent gallery-dl sidecar
|
||||
(if any) is applied via _apply_sidecar — operator-flagged
|
||||
2026-05-25: scanning a GS download dir with smaller IR-migrated
|
||||
images on the receiving end used to swap files but lose the GS
|
||||
sidecar's post metadata entirely. _apply_sidecar is additive
|
||||
(find-or-create Post / Source / ImageProvenance, NULL-only
|
||||
primary_post_id update) so any pre-existing Post linkage
|
||||
survives untouched.
|
||||
|
||||
If `new_path` is provided, `source` is assumed to ALREADY be at
|
||||
that path (FC-3c attach_in_place case) — skip the copy step.
|
||||
Otherwise the file is copied via _copy_to_library."""
|
||||
@@ -750,6 +981,26 @@ class Importer:
|
||||
# created_at intentionally preserved; updated_at auto-bumps.
|
||||
self.session.flush()
|
||||
self.session.commit()
|
||||
# The phash candidate cache (used to avoid N+1 selects during
|
||||
# archive imports) is now stale for `existing.id` — the row's
|
||||
# phash/dimensions changed. Invalidate; the next call re-fetches.
|
||||
self._phash_candidates = None
|
||||
|
||||
# Sidecar enrichment from the new (larger) file's location.
|
||||
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||
# existing row has none, and is internally guarded against
|
||||
# missing-or-malformed sidecars (silent return).
|
||||
try:
|
||||
self._apply_sidecar(existing, source, None)
|
||||
except Exception as exc:
|
||||
# Don't unwind the supersede DB swap if sidecar parsing
|
||||
# blows up unexpectedly — the file replacement is the
|
||||
# critical operation, sidecar is enrichment.
|
||||
log.warning(
|
||||
"sidecar enrichment failed during supersede of "
|
||||
"image_record.id=%s from %s: %s",
|
||||
existing.id, source, exc,
|
||||
)
|
||||
|
||||
for stale in (old_path, old_thumb):
|
||||
if not stale or stale == str(dest):
|
||||
@@ -765,8 +1016,26 @@ class Importer:
|
||||
pass
|
||||
|
||||
def _transparency_pct(self, source: Path) -> float:
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||
|
||||
For animated formats (multi-frame WebP / GIF / APNG), short-circuit
|
||||
to 0.0 instead of decoding every frame. PIL's `getchannel("A")`
|
||||
forces a full decode of all frames in an animated image, which for
|
||||
a large animated WebP takes 5+ minutes and blows past the Celery
|
||||
soft+hard time limits (300s/360s → SIGKILL). Operator-flagged
|
||||
2026-05-26. Transparency analysis on a multi-frame image isn't
|
||||
meaningful for art-curation purposes anyway — different frames
|
||||
have different alpha — so the existing too_transparent skip rule
|
||||
is bypassed entirely for animated content.
|
||||
"""
|
||||
with Image.open(source) as im:
|
||||
if getattr(im, "is_animated", False):
|
||||
log.info(
|
||||
"skipping transparency check for animated image %s "
|
||||
"(n_frames=%d) — avoids multi-frame decode timeout",
|
||||
source, getattr(im, "n_frames", 0),
|
||||
)
|
||||
return 0.0
|
||||
if im.mode not in ("RGBA", "LA") and not (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
):
|
||||
|
||||
@@ -34,10 +34,21 @@ class Embedder:
|
||||
if self._model is not None:
|
||||
return
|
||||
import torch
|
||||
from transformers import AutoModel, AutoProcessor
|
||||
from transformers import AutoModel, SiglipImageProcessor
|
||||
|
||||
self._torch = torch
|
||||
self._processor = AutoProcessor.from_pretrained(str(self._model_dir))
|
||||
# FC's embedder only does IMAGE inference — never text. AutoProcessor
|
||||
# loads the full processor including SiglipTokenizer, which requires
|
||||
# the sentencepiece library at import time even if we never call it.
|
||||
# SiglipImageProcessor loads ONLY preprocessor_config.json (image
|
||||
# side) and skips the tokenizer config entirely. Operator hit the
|
||||
# ImportError 2026-05-25 once the ml-worker started actually running
|
||||
# tag_and_embed; switching to the image-only loader avoids the
|
||||
# tokenizer dep without adding ~30 MB of unused C++ build to the
|
||||
# lean ml-worker image.
|
||||
self._processor = SiglipImageProcessor.from_pretrained(
|
||||
str(self._model_dir)
|
||||
)
|
||||
self._model = AutoModel.from_pretrained(str(self._model_dir))
|
||||
self._model.eval()
|
||||
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
"""FC-3b platforms registry — the single source of truth for what
|
||||
FabledCurator supports.
|
||||
|
||||
Lifted from GallerySubscriber's
|
||||
~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py
|
||||
and ~/.../extension/lib/platforms.js. Six platforms; auth_type and
|
||||
URL patterns match GS exactly so the existing browser extension
|
||||
hits FC unmodified.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlatformInfo:
|
||||
key: str
|
||||
name: str
|
||||
description: str
|
||||
auth_type: Literal["cookies", "token"]
|
||||
requires_auth: bool
|
||||
url_pattern: str
|
||||
url_examples: list[str]
|
||||
default_config: dict
|
||||
notes: str | None = None
|
||||
|
||||
|
||||
# Common defaults used across most platforms; embedded per-platform
|
||||
# below so per-platform overrides remain explicit.
|
||||
_DEFAULTS = {
|
||||
"sleep": 3.0,
|
||||
"sleep_request": 1.5,
|
||||
"skip_existing": True,
|
||||
"save_metadata": True,
|
||||
"timeout": 3600,
|
||||
}
|
||||
|
||||
|
||||
PLATFORMS: dict[str, PlatformInfo] = {
|
||||
"patreon": PlatformInfo(
|
||||
key="patreon",
|
||||
name="Patreon",
|
||||
description="Download posts from Patreon creators",
|
||||
auth_type="cookies",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?patreon\.com/",
|
||||
url_examples=[
|
||||
"https://www.patreon.com/example_artist",
|
||||
"https://www.patreon.com/user?u=12345678",
|
||||
],
|
||||
default_config={**_DEFAULTS, "content_types": ["images", "attachments"]},
|
||||
),
|
||||
"subscribestar": PlatformInfo(
|
||||
key="subscribestar",
|
||||
name="SubscribeStar",
|
||||
description="Download posts from SubscribeStar creators",
|
||||
auth_type="cookies",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/",
|
||||
url_examples=[
|
||||
"https://subscribestar.adult/example_artist",
|
||||
"https://www.subscribestar.com/example_artist",
|
||||
],
|
||||
default_config={**_DEFAULTS, "content_types": ["all"]},
|
||||
),
|
||||
"hentaifoundry": PlatformInfo(
|
||||
key="hentaifoundry",
|
||||
name="Hentai Foundry",
|
||||
description="Download artwork from Hentai Foundry artists",
|
||||
auth_type="cookies",
|
||||
requires_auth=False,
|
||||
url_pattern=r"^https?://(www\.)?hentai-foundry\.com/",
|
||||
url_examples=[
|
||||
"https://www.hentai-foundry.com/user/example_artist",
|
||||
"https://www.hentai-foundry.com/pictures/user/example_artist",
|
||||
],
|
||||
default_config={**_DEFAULTS, "content_types": ["pictures"]},
|
||||
),
|
||||
"discord": PlatformInfo(
|
||||
key="discord",
|
||||
name="Discord",
|
||||
description="Download attachments from Discord channels",
|
||||
auth_type="token",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?discord\.com/channels/",
|
||||
url_examples=["https://discord.com/channels/123456789/987654321"],
|
||||
default_config={**_DEFAULTS, "content_types": ["all"]},
|
||||
notes="Requires Discord user token (not bot token).",
|
||||
),
|
||||
"pixiv": PlatformInfo(
|
||||
key="pixiv",
|
||||
name="Pixiv",
|
||||
description="Download artwork from Pixiv artists",
|
||||
auth_type="token",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?pixiv\.net/",
|
||||
url_examples=[
|
||||
"https://www.pixiv.net/users/12345678",
|
||||
"https://www.pixiv.net/en/users/12345678",
|
||||
],
|
||||
default_config={**_DEFAULTS, "content_types": ["all"]},
|
||||
notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.",
|
||||
),
|
||||
"deviantart": PlatformInfo(
|
||||
key="deviantart",
|
||||
name="DeviantArt",
|
||||
description="Download artwork from DeviantArt artists",
|
||||
auth_type="cookies",
|
||||
requires_auth=False,
|
||||
url_pattern=r"^https?://(www\.)?deviantart\.com/",
|
||||
url_examples=[
|
||||
"https://www.deviantart.com/example-artist",
|
||||
"https://www.deviantart.com/example-artist/gallery",
|
||||
],
|
||||
default_config={**_DEFAULTS, "content_types": ["gallery"]},
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def known_platform_keys() -> frozenset[str]:
|
||||
return frozenset(PLATFORMS.keys())
|
||||
|
||||
|
||||
def auth_type_for(platform: str) -> str | None:
|
||||
info = PLATFORMS.get(platform)
|
||||
return info.auth_type if info else None
|
||||
|
||||
|
||||
def to_dict(info: PlatformInfo) -> dict:
|
||||
return {
|
||||
"key": info.key,
|
||||
"name": info.name,
|
||||
"description": info.description,
|
||||
"auth_type": info.auth_type,
|
||||
"requires_auth": info.requires_auth,
|
||||
"url_pattern": info.url_pattern,
|
||||
"url_examples": info.url_examples,
|
||||
"default_config": info.default_config,
|
||||
"notes": info.notes,
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
"""FC-3b platforms registry — single source of truth for what
|
||||
FabledCurator supports + where each platform's quirks live.
|
||||
|
||||
Adding a new platform: drop a new module `<platform>.py` next to this
|
||||
one, declare an `INFO = PlatformInfo(...)`, add the import + entry in
|
||||
PLATFORMS below. Sidecar parsing, cookie materialization, and
|
||||
`/api/platforms` pick it up automatically.
|
||||
|
||||
Lifted from GallerySubscriber's
|
||||
~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py
|
||||
and ~/.../extension/lib/platforms.js. Six platforms; auth_type and
|
||||
URL patterns match GS exactly so the existing browser extension
|
||||
hits FC unmodified.
|
||||
"""
|
||||
|
||||
from .base import (
|
||||
DEFAULT_DESCRIPTION_KEYS,
|
||||
DEFAULT_EXTERNAL_POST_ID_KEYS,
|
||||
PlatformInfo,
|
||||
)
|
||||
from .deviantart import INFO as _DEVIANTART
|
||||
from .discord import INFO as _DISCORD
|
||||
from .hentaifoundry import INFO as _HENTAIFOUNDRY
|
||||
from .patreon import INFO as _PATREON
|
||||
from .pixiv import INFO as _PIXIV
|
||||
from .subscribestar import INFO as _SUBSCRIBESTAR
|
||||
|
||||
PLATFORMS: dict[str, PlatformInfo] = {
|
||||
info.key: info
|
||||
for info in (
|
||||
_PATREON,
|
||||
_SUBSCRIBESTAR,
|
||||
_HENTAIFOUNDRY,
|
||||
_DISCORD,
|
||||
_PIXIV,
|
||||
_DEVIANTART,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def known_platform_keys() -> frozenset[str]:
|
||||
return frozenset(PLATFORMS.keys())
|
||||
|
||||
|
||||
def auth_type_for(platform: str) -> str | None:
|
||||
info = PLATFORMS.get(platform)
|
||||
return info.auth_type if info else None
|
||||
|
||||
|
||||
def to_dict(info: PlatformInfo) -> dict:
|
||||
"""Serialize a PlatformInfo to a JSON-safe dict for /api/platforms.
|
||||
|
||||
Behavioral fields (callables, sidecar-chain overrides) are
|
||||
intentionally omitted — they aren't useful to API consumers.
|
||||
"""
|
||||
return {
|
||||
"key": info.key,
|
||||
"name": info.name,
|
||||
"description": info.description,
|
||||
"auth_type": info.auth_type,
|
||||
"requires_auth": info.requires_auth,
|
||||
"url_pattern": info.url_pattern,
|
||||
"url_examples": info.url_examples,
|
||||
"default_config": info.default_config,
|
||||
"notes": info.notes,
|
||||
}
|
||||
|
||||
|
||||
def external_post_id_keys_for(platform: str | None) -> tuple[str, ...]:
|
||||
"""Resolve the external_post_id lookup chain for a given platform,
|
||||
falling back to the module default when the platform isn't
|
||||
registered or hasn't overridden the chain."""
|
||||
info = PLATFORMS.get(platform) if platform else None
|
||||
if info is not None and info.external_post_id_keys is not None:
|
||||
return info.external_post_id_keys
|
||||
return DEFAULT_EXTERNAL_POST_ID_KEYS
|
||||
|
||||
|
||||
def description_keys_for(platform: str | None) -> tuple[str, ...]:
|
||||
"""Resolve the description body lookup chain for a given platform."""
|
||||
info = PLATFORMS.get(platform) if platform else None
|
||||
if info is not None and info.description_keys is not None:
|
||||
return info.description_keys
|
||||
return DEFAULT_DESCRIPTION_KEYS
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PLATFORMS",
|
||||
"PlatformInfo",
|
||||
"auth_type_for",
|
||||
"description_keys_for",
|
||||
"external_post_id_keys_for",
|
||||
"known_platform_keys",
|
||||
"to_dict",
|
||||
]
|
||||
@@ -0,0 +1,105 @@
|
||||
"""PlatformInfo dataclass + shared defaults + small helpers.
|
||||
|
||||
Per-platform modules import from here, register their PlatformInfo via
|
||||
INFO, optionally attaching `derive_post_url` and/or `augment_cookies`
|
||||
callables for behavior that diverges from gallery-dl's mainline shape
|
||||
(Patreon).
|
||||
|
||||
Adding a new platform: drop a new module under `services/platforms/`,
|
||||
declare an INFO, and add it to the import list in
|
||||
`services/platforms/__init__.py`. Sidecar parsing, cookie
|
||||
materialization, and the /api/platforms response pick it up
|
||||
automatically.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
# Sidecar parsing defaults. Per-platform PlatformInfo entries can
|
||||
# override these by setting `external_post_id_keys=` /
|
||||
# `description_keys=`. Most don't need to — the defaults already cover
|
||||
# every platform FC supports.
|
||||
#
|
||||
# external_post_id chain: `post_id` MUST come before `id` because
|
||||
# SubscribeStar gallery-dl puts the per-attachment id in `id` and the
|
||||
# actual post id in `post_id`; picking `id` first fragments
|
||||
# multi-image SubscribeStar posts into N Post rows. Patreon/Pixiv have
|
||||
# no `post_id` so `id` still wins for them; HF uses `index`, Discord
|
||||
# uses `message_id` — all reached via the remaining chain entries.
|
||||
# (Banked 2026-05-27 during the sidecar audit.)
|
||||
DEFAULT_EXTERNAL_POST_ID_KEYS: tuple[str, ...] = (
|
||||
"post_id", "id", "index", "message_id",
|
||||
)
|
||||
|
||||
# Description body chain: Discord's gallery-dl extractor uses `message`
|
||||
# (no `content`); appended to the chain so Discord posts surface body
|
||||
# text.
|
||||
DEFAULT_DESCRIPTION_KEYS: tuple[str, ...] = (
|
||||
"content", "description", "caption", "message",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlatformInfo:
|
||||
# --- Identity / metadata ---
|
||||
key: str
|
||||
name: str
|
||||
description: str
|
||||
auth_type: Literal["cookies", "token"]
|
||||
requires_auth: bool
|
||||
url_pattern: str
|
||||
url_examples: list[str]
|
||||
default_config: dict
|
||||
notes: str | None = None
|
||||
|
||||
# --- Sidecar parsing overrides ---
|
||||
# Each is None to mean "use the module default above"; a platform
|
||||
# only sets one of these when its sidecar shape genuinely differs.
|
||||
external_post_id_keys: tuple[str, ...] | None = None
|
||||
description_keys: tuple[str, ...] | None = None
|
||||
|
||||
# --- Behavioral hooks ---
|
||||
# Synthesize a post permalink from sidecar data. Required when
|
||||
# gallery-dl's `url` field is the file/CDN URL rather than the post
|
||||
# permalink (subscribestar/pixiv/hf/discord). None = trust the bare
|
||||
# `url` field (patreon, deviantart).
|
||||
derive_post_url: Callable[[dict], str | None] | None = None
|
||||
|
||||
# Post-process the materialized cookies.txt for gallery-dl. Used by
|
||||
# platforms whose server gates or extractor quirks need synthetic
|
||||
# cookies the extension can't capture (subscribestar age cookie, HF
|
||||
# host-only PHPSESSID duplicate). None = no-op.
|
||||
augment_cookies: Callable[[str], str] | None = None
|
||||
|
||||
|
||||
def str_id_value(v) -> str | None:
|
||||
"""Coerce a JSON scalar id into a non-empty string, rejecting bool
|
||||
(Python's bool is an int subclass so `isinstance(True, int)` is
|
||||
True; without this guard a sidecar with `"id": true` would produce
|
||||
external_post_id="True")."""
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
if isinstance(v, (str, int)) and str(v).strip():
|
||||
return str(v).strip()
|
||||
return None
|
||||
|
||||
|
||||
def str_field(v) -> str | None:
|
||||
"""Same idea as str_id_value but for plain string fields (no int
|
||||
coercion)."""
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
# Shared gallery-dl invocation defaults. Embedded in each platform's
|
||||
# default_config (with platform-specific overrides) so per-platform
|
||||
# choices stay explicit.
|
||||
GD_DEFAULTS = {
|
||||
"sleep": 3.0,
|
||||
"sleep_request": 1.5,
|
||||
"skip_existing": True,
|
||||
"save_metadata": True,
|
||||
"timeout": 3600,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""DeviantArt — no exercised quirks yet.
|
||||
|
||||
No operator-owned DeviantArt archive existed at the 2026-05-27 sidecar
|
||||
audit, so we don't know yet whether DA's gallery-dl sidecars are
|
||||
well-behaved or have their own quirks. When DA gets exercised for the
|
||||
first time, add `derive_post_url` / `augment_cookies` here as needed.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="deviantart",
|
||||
name="DeviantArt",
|
||||
description="Download artwork from DeviantArt artists",
|
||||
auth_type="cookies",
|
||||
requires_auth=False,
|
||||
url_pattern=r"^https?://(www\.)?deviantart\.com/",
|
||||
url_examples=[
|
||||
"https://www.deviantart.com/example-artist",
|
||||
"https://www.deviantart.com/example-artist/gallery",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["gallery"]},
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Discord — one quirk + one already-default.
|
||||
|
||||
post_url: gallery-dl's `url` is the CDN attachment URL. The "permalink"
|
||||
for a Discord message uses the (server, channel, message) triple via
|
||||
`discord.com/channels/<server>/<channel>/<message>`. Note that
|
||||
permalinks are only resolvable for users in the same server — public
|
||||
access doesn't work — but the URL is still useful to the operator
|
||||
in-app.
|
||||
|
||||
Description body is in `message` not `content`. That's already covered
|
||||
by the default description chain in base.py (DEFAULT_DESCRIPTION_KEYS
|
||||
ends with `message`). No description_keys override needed.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
|
||||
|
||||
def derive_post_url(data: dict) -> str | None:
|
||||
sid = str_id_value(data.get("server_id"))
|
||||
cid = str_id_value(data.get("channel_id"))
|
||||
mid = str_id_value(data.get("message_id"))
|
||||
if sid and cid and mid:
|
||||
return f"https://discord.com/channels/{sid}/{cid}/{mid}"
|
||||
return None
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="discord",
|
||||
name="Discord",
|
||||
description="Download attachments from Discord channels",
|
||||
auth_type="token",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?discord\.com/channels/",
|
||||
url_examples=["https://discord.com/channels/123456789/987654321"],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["all"]},
|
||||
notes="Requires Discord user token (not bot token).",
|
||||
derive_post_url=derive_post_url,
|
||||
)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""HentaiFoundry — two quirks colocated.
|
||||
|
||||
1. post_url: HF sidecars omit `url` entirely; `src` is the image URL.
|
||||
Synthesize the permalink from `user` + `index`
|
||||
(/pictures/user/<user>/<index>).
|
||||
|
||||
2. augment_cookies: gallery-dl's HF extractor checks
|
||||
`self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with
|
||||
`requests`' EXACT domain matching. The extension's pre-v1.0.5
|
||||
`cookies.js` aggressively rewrote every captured cookie to the
|
||||
leading-dot subdomain-wide form (`.hentai-foundry.com`), which fails
|
||||
the exact lookup even though the cookie IS sent on actual HTTP
|
||||
requests (RFC 6265 subdomain matching). The extractor falls into
|
||||
an unauthenticated `?enterAgree=1` HEAD that 401s. Inject host-only
|
||||
duplicates of PHPSESSID + YII_CSRF_TOKEN so the lookup succeeds.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_field, str_id_value
|
||||
|
||||
_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN")
|
||||
|
||||
|
||||
def derive_post_url(data: dict) -> str | None:
|
||||
user = str_field(data.get("user")) or str_field(data.get("artist"))
|
||||
idx = str_id_value(data.get("index"))
|
||||
if user and idx:
|
||||
return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}"
|
||||
return None
|
||||
|
||||
|
||||
def augment_cookies(netscape: str) -> str:
|
||||
body = netscape.rstrip("\n")
|
||||
if not body:
|
||||
return netscape
|
||||
lines = body.split("\n")
|
||||
existing_host_only: set[str] = set()
|
||||
by_name: dict[str, list[str]] = {}
|
||||
for raw in lines:
|
||||
if not raw or raw.startswith("#"):
|
||||
continue
|
||||
parts = raw.split("\t")
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
domain, _flag, _path, _secure, _exp, name, _value = parts[:7]
|
||||
if name not in _HOST_ONLY_NAMES:
|
||||
continue
|
||||
if domain == "www.hentai-foundry.com":
|
||||
existing_host_only.add(name)
|
||||
elif domain in (".hentai-foundry.com", "hentai-foundry.com"):
|
||||
by_name.setdefault(name, []).append(raw)
|
||||
|
||||
appended: list[str] = []
|
||||
for name in _HOST_ONLY_NAMES:
|
||||
if name in existing_host_only or name not in by_name:
|
||||
continue
|
||||
# Duplicate the first subdomain-wide line as host-only on
|
||||
# www.hentai-foundry.com. Same value + expiry; flag=FALSE marks
|
||||
# the entry host-only in netscape format.
|
||||
parts = by_name[name][0].split("\t")
|
||||
parts[0] = "www.hentai-foundry.com"
|
||||
parts[1] = "FALSE"
|
||||
appended.append("\t".join(parts[:7]))
|
||||
|
||||
if not appended:
|
||||
return netscape
|
||||
return body + "\n" + "\n".join(appended) + "\n"
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="hentaifoundry",
|
||||
name="Hentai Foundry",
|
||||
description="Download artwork from Hentai Foundry artists",
|
||||
auth_type="cookies",
|
||||
requires_auth=False,
|
||||
url_pattern=r"^https?://(www\.)?hentai-foundry\.com/",
|
||||
url_examples=[
|
||||
"https://www.hentai-foundry.com/user/example_artist",
|
||||
"https://www.hentai-foundry.com/pictures/user/example_artist",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["pictures"]},
|
||||
derive_post_url=derive_post_url,
|
||||
augment_cookies=augment_cookies,
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Patreon — no quirks. The reference platform.
|
||||
|
||||
Patreon's gallery-dl sidecars are the well-behaved baseline: `url` is a
|
||||
real permalink, `id` is the post id, `title` and `content` are
|
||||
populated. No cookie quirks (session cookies are domain-wide). No
|
||||
derivation overrides.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="patreon",
|
||||
name="Patreon",
|
||||
description="Download posts from Patreon creators",
|
||||
auth_type="cookies",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?patreon\.com/",
|
||||
url_examples=[
|
||||
"https://www.patreon.com/example_artist",
|
||||
"https://www.patreon.com/user?u=12345678",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["images", "attachments"]},
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Pixiv — one quirk.
|
||||
|
||||
post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The
|
||||
post permalink follows /artworks/<id>. external_post_id (= `id`) was
|
||||
already correct, so no override there.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
|
||||
|
||||
def derive_post_url(data: dict) -> str | None:
|
||||
pid = str_id_value(data.get("id"))
|
||||
if pid:
|
||||
return f"https://www.pixiv.net/artworks/{pid}"
|
||||
return None
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="pixiv",
|
||||
name="Pixiv",
|
||||
description="Download artwork from Pixiv artists",
|
||||
auth_type="token",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?pixiv\.net/",
|
||||
url_examples=[
|
||||
"https://www.pixiv.net/users/12345678",
|
||||
"https://www.pixiv.net/en/users/12345678",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["all"]},
|
||||
notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.",
|
||||
derive_post_url=derive_post_url,
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""SubscribeStar — three quirks colocated.
|
||||
|
||||
1. external_post_id: gallery-dl puts the per-attachment id in `id`
|
||||
(e.g. 711509) and the actual post id in `post_id` (e.g. 360360).
|
||||
The default chain in base.py already prefers `post_id`; this module
|
||||
doesn't need to override it but the comment lives here too so a
|
||||
future reader knows the chain's order was driven by this platform.
|
||||
|
||||
2. post_url: gallery-dl's `url` is the file CDN URL
|
||||
(`/post_uploads?payload=...`). Synthesize the post permalink from
|
||||
`post_id`.
|
||||
|
||||
3. augment_cookies: the server gates artist pages behind a
|
||||
`_personalization_id` age-confirmation cookie that the user can't
|
||||
easily refresh — SubscribeStar's frontend JS uses localStorage to
|
||||
suppress the age popup once dismissed. gallery-dl's own login flow
|
||||
sidesteps this by setting `18_plus_agreement_generic=true` on
|
||||
`.subscribestar.adult`; we mirror that for cookies captured via the
|
||||
extension.
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
|
||||
|
||||
def derive_post_url(data: dict) -> str | None:
|
||||
pid = str_id_value(data.get("post_id"))
|
||||
if pid:
|
||||
return f"https://www.subscribestar.com/posts/{pid}"
|
||||
return None
|
||||
|
||||
|
||||
def augment_cookies(netscape: str) -> str:
|
||||
if "18_plus_agreement_generic" in netscape:
|
||||
return netscape
|
||||
# Far-future expiry — gallery-dl's own login flow sets this with no
|
||||
# explicit expiry; the server only checks presence/value.
|
||||
expiry = 4102444800 # 2100-01-01 UTC
|
||||
line = "\t".join([
|
||||
".subscribestar.adult", "TRUE", "/", "TRUE",
|
||||
str(expiry), "18_plus_agreement_generic", "true",
|
||||
])
|
||||
body = netscape.rstrip("\n")
|
||||
if not body:
|
||||
body = "# Netscape HTTP Cookie File"
|
||||
return body + "\n" + line + "\n"
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="subscribestar",
|
||||
name="SubscribeStar",
|
||||
description="Download posts from SubscribeStar creators",
|
||||
auth_type="cookies",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/",
|
||||
url_examples=[
|
||||
"https://subscribestar.adult/example_artist",
|
||||
"https://www.subscribestar.com/example_artist",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["all"]},
|
||||
derive_post_url=derive_post_url,
|
||||
augment_cookies=augment_cookies,
|
||||
)
|
||||
@@ -109,7 +109,10 @@ class PostFeedService:
|
||||
if row is None:
|
||||
return None
|
||||
post, artist, source = row
|
||||
thumbs_map = await self._thumbnails_for([post.id])
|
||||
# Detail endpoint returns the FULL image list for PostModal's
|
||||
# masonry grid — feed query still caps at THUMBNAIL_LIMIT via
|
||||
# the default arg.
|
||||
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
||||
atts_map = await self._attachments_for([post.id])
|
||||
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
||||
item["description_full"] = html_to_plain(post.description)
|
||||
@@ -117,15 +120,21 @@ class PostFeedService:
|
||||
|
||||
# --- composition helpers ---------------------------------------------
|
||||
|
||||
async def _thumbnails_for(self, post_ids: list[int]) -> dict[int, dict]:
|
||||
"""post_id -> {"thumbs": [...up to 6], "more": int}.
|
||||
async def _thumbnails_for(
|
||||
self, post_ids: list[int], *, limit: int | None = THUMBNAIL_LIMIT,
|
||||
) -> dict[int, dict]:
|
||||
"""post_id -> {"thumbs": [...up to limit], "more": int}.
|
||||
|
||||
Selects THUMBNAIL_LIMIT+1 images per post via window function so we
|
||||
can detect overflow in a single query.
|
||||
Selects up to `limit` images per post via window function so we
|
||||
can detect overflow in a single query. Pass `limit=None` to
|
||||
return ALL thumbnails per post (used by `get_post` for PostModal's
|
||||
masonry grid; the feed pass keeps the default cap so payloads
|
||||
stay small).
|
||||
"""
|
||||
if not post_ids:
|
||||
return {}
|
||||
# Rank images within each post and fetch only the top THUMBNAIL_LIMIT+1.
|
||||
# Rank images within each post; cap at `limit` rows per post when
|
||||
# limit is set, return all when limit is None.
|
||||
ranked = (
|
||||
select(
|
||||
ImageRecord.id,
|
||||
@@ -143,12 +152,13 @@ class PostFeedService:
|
||||
.where(ImageRecord.primary_post_id.in_(post_ids))
|
||||
.subquery()
|
||||
)
|
||||
rows = (await self.session.execute(
|
||||
select(
|
||||
ranked.c.id, ranked.c.primary_post_id,
|
||||
ranked.c.sha256, ranked.c.mime, ranked.c.total,
|
||||
).where(ranked.c.rn <= THUMBNAIL_LIMIT)
|
||||
)).all()
|
||||
stmt = select(
|
||||
ranked.c.id, ranked.c.primary_post_id,
|
||||
ranked.c.sha256, ranked.c.mime, ranked.c.total,
|
||||
)
|
||||
if limit is not None:
|
||||
stmt = stmt.where(ranked.c.rn <= limit)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
|
||||
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
||||
for img_id, pid, sha, mime, total in rows:
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Layer-2 one-shot re-download remediation for corrupt imported files.
|
||||
|
||||
When an import fails on a file that came from a known, pollable
|
||||
subscription Source, deleting the bad copy and re-running the source's
|
||||
downloader can fetch a fresh, unblemished copy. This only helps when:
|
||||
|
||||
- the corruption is in transit / on disk (not at the source), AND
|
||||
- the file resolves to an ENABLED Source with a real feed URL
|
||||
(a `sidecar:<platform>:<slug>` synthetic anchor is not pollable),
|
||||
AND
|
||||
- we haven't already re-fetched this task once (bounded by
|
||||
ImportTask.refetched so source-side corruption can't loop).
|
||||
|
||||
Filesystem-only imports with no resolvable Source return 'no_source' —
|
||||
the operator's only remediation there is to replace the file on disk.
|
||||
|
||||
Operator-requested 2026-05-28 (Layer 2).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, ImportTask, Source
|
||||
from ..utils.paths import derive_top_level_artist
|
||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from ..utils.slug import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_refetch_source(
|
||||
session: Session, source_path: str, import_root: Path,
|
||||
) -> Source | None:
|
||||
"""Find an enabled, real-URL Source for the file's (artist, platform),
|
||||
or None when nothing re-pollable resolves."""
|
||||
path = Path(source_path)
|
||||
sc = find_sidecar(path)
|
||||
if sc is None:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(sc.read_text("utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
if not sd.platform:
|
||||
return None
|
||||
artist_name = derive_top_level_artist(path, import_root)
|
||||
if not artist_name:
|
||||
return None
|
||||
artist = session.execute(
|
||||
select(Artist).where(Artist.slug == slugify(artist_name))
|
||||
).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return None
|
||||
src = session.execute(
|
||||
select(Source)
|
||||
.where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == sd.platform,
|
||||
Source.enabled.is_(True),
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
).scalars().first()
|
||||
if src is None:
|
||||
return None
|
||||
if (src.url or "").startswith("sidecar:"):
|
||||
return None # synthetic anchor — not a pollable feed
|
||||
return src
|
||||
|
||||
|
||||
def attempt_refetch(
|
||||
session: Session, task: ImportTask, import_root: Path,
|
||||
) -> dict:
|
||||
"""Delete the corrupt file, mark the task refetched, and trigger ONE
|
||||
source re-check. Idempotent/bounded: a task already refetched (or
|
||||
with no resolvable Source) is a no-op. Commits."""
|
||||
if task.refetched:
|
||||
return {"status": "already_refetched"}
|
||||
src = resolve_refetch_source(session, task.source_path, import_root)
|
||||
if src is None:
|
||||
return {"status": "no_source"}
|
||||
|
||||
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
||||
# the source re-check instead of skipping the still-present corrupt
|
||||
# file.
|
||||
try:
|
||||
Path(task.source_path).unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
log.warning("refetch unlink failed for %s: %s", task.source_path, exc)
|
||||
|
||||
task.refetched = True
|
||||
session.add(task)
|
||||
session.commit()
|
||||
|
||||
# Lazy import to avoid a tasks→services→tasks import cycle at module
|
||||
# load. download_source.delay() is sync-safe in any context.
|
||||
from ..tasks.download import download_source
|
||||
|
||||
download_source.delay(src.id)
|
||||
return {"status": "refetch_queued", "source_id": src.id}
|
||||
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
|
||||
def _map_result_to_status(result):
|
||||
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
||||
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb."""
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb.
|
||||
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
|
||||
complete, no ML/thumb re-derive (file/pixels unchanged)."""
|
||||
if result.status in ("imported", "superseded"):
|
||||
return ("complete", True)
|
||||
if result.status == "attached":
|
||||
if result.status in ("attached", "refreshed"):
|
||||
return ("complete", False)
|
||||
if result.status == "skipped":
|
||||
return ("skipped", False)
|
||||
@@ -62,30 +64,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.import_file.import_media_file",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=300,
|
||||
time_limit=360,
|
||||
)
|
||||
def import_media_file(self, import_task_id: int) -> dict:
|
||||
"""Returns a dict so the eager-mode tests can assert without DB.
|
||||
|
||||
Decorator notes:
|
||||
- autoretry_for: transient DB / filesystem errors retry with
|
||||
exponential backoff (5s base, jitter, max 3 attempts). On final
|
||||
give-up the task raises and acks_late=True (set globally on the
|
||||
Celery app) does NOT redeliver — the recovery sweep catches the
|
||||
row instead.
|
||||
- soft_time_limit (300s) raises SoftTimeLimitExceeded in this
|
||||
process so the task can mark its row failed before being killed.
|
||||
- time_limit (360s) is the hard cap; SIGKILL if the soft signal
|
||||
was swallowed.
|
||||
def _run_import_task(import_task_id: int) -> dict:
|
||||
"""Shared body for import_media_file + import_archive_file. The two
|
||||
tasks differ ONLY in their Celery time limits (a single media file
|
||||
is sub-second; an archive runs the full per-member pipeline inline
|
||||
for every member and can take many minutes). Both flip the row to
|
||||
'processing', dispatch to `_do_import`, and honor the
|
||||
flip-to-terminal resilience contract.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
@@ -101,19 +86,85 @@ def import_media_file(self, import_task_id: int) -> dict:
|
||||
try:
|
||||
return _do_import(session, task, import_task_id)
|
||||
except SoftTimeLimitExceeded:
|
||||
_mark_failed(session, task, "soft_time_limit exceeded (>300s)")
|
||||
_mark_failed(session, task, "soft_time_limit exceeded")
|
||||
raise
|
||||
except (OperationalError, DBAPIError, OSError):
|
||||
# Retryable per the decorator; do NOT mark failed (let
|
||||
# autoretry have a clean go at it). If autoretry exhausts,
|
||||
# the row stays 'processing' and the maintenance sweep
|
||||
# flips it within 5 min.
|
||||
# flips it.
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
||||
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
||||
raise
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.import_file.import_media_file",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=300,
|
||||
time_limit=360,
|
||||
)
|
||||
def import_media_file(self, import_task_id: int) -> dict:
|
||||
"""Import ONE media file (or non-media → PostAttachment). Sub-second
|
||||
for the common case; the tight 5-min soft limit keeps a genuinely
|
||||
stuck single-file import detectable fast.
|
||||
|
||||
Decorator notes:
|
||||
- autoretry_for: transient DB / filesystem errors retry with
|
||||
exponential backoff (5s base, jitter, max 3 attempts). On final
|
||||
give-up the task raises and acks_late=True (set globally on the
|
||||
Celery app) does NOT redeliver — the recovery sweep catches the
|
||||
row instead.
|
||||
- soft_time_limit (300s) raises SoftTimeLimitExceeded in-process
|
||||
so the task can mark its row failed before being killed.
|
||||
- time_limit (360s) is the hard SIGKILL cap.
|
||||
"""
|
||||
return _run_import_task(import_task_id)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.import_file.import_archive_file",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
# Archives run the full per-member pipeline (sha256 + pHash + dedup
|
||||
# query + copy + provenance) for EVERY media member inline, under a
|
||||
# single task budget. A multi-hundred-member archive blows the
|
||||
# 5-min media limit. soft=30min / hard=35min sizes for a large
|
||||
# archive. Operator-flagged 2026-05-28 (target 1645019 hit the old
|
||||
# shared 300s soft limit). The recovery sweep gives this task its
|
||||
# own 40-min threshold via maintenance.TASK_STUCK_THRESHOLD_MINUTES
|
||||
# so it isn't preempted while legitimately grinding through members.
|
||||
soft_time_limit=1800,
|
||||
time_limit=2100,
|
||||
)
|
||||
def import_archive_file(self, import_task_id: int) -> dict:
|
||||
"""Import an archive: extract + run the per-member media pipeline for
|
||||
every member inline, then preserve the archive as a PostAttachment.
|
||||
Same body as import_media_file (dispatch is by file kind inside
|
||||
Importer.import_one); split out purely for the larger time budget."""
|
||||
return _run_import_task(import_task_id)
|
||||
|
||||
|
||||
def enqueue_import(task_id: int, task_type: str) -> None:
|
||||
"""Route an ImportTask to the right Celery task by its task_type.
|
||||
Single source of truth for the media-vs-archive dispatch so the
|
||||
scan, retry, and recovery-requeue paths stay in sync."""
|
||||
if task_type == "archive":
|
||||
import_archive_file.delay(task_id)
|
||||
else:
|
||||
import_media_file.delay(task_id)
|
||||
|
||||
|
||||
def _do_import(session, task, import_task_id: int) -> dict:
|
||||
"""Actual work, called from inside the resilience wrapper."""
|
||||
settings = session.execute(
|
||||
@@ -138,6 +189,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "imported"
|
||||
counter_col = ImportBatch.imported
|
||||
elif result.status == "refreshed":
|
||||
# Deep-scan rederive: existing row got phash/artist/sidecar
|
||||
# refreshed. Task is complete (no further work), but counted in
|
||||
# `refreshed` not `imported` so the UI can surface the actual
|
||||
# work done. operator-flagged 2026-05-25.
|
||||
task.status = "complete"
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "refreshed"
|
||||
counter_col = ImportBatch.refreshed
|
||||
elif result.status == "attached":
|
||||
task.status = "complete"
|
||||
counter_col_name = "attachments"
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
"""scan_library_for_rule Celery task — iterates image_record in keyset-
|
||||
paginated batches, evaluates the audit rule per image, populates
|
||||
LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft
|
||||
time limit (plenty of margin for 100k+ image libraries at ~100ms PIL
|
||||
decode + histogram per image).
|
||||
|
||||
State machine:
|
||||
start: status='running'
|
||||
end success: status='ready'
|
||||
end error: status='error', error=traceback
|
||||
oversize: status='error', error='matched too many images; tighten threshold'
|
||||
external cancel: scan sees status='cancelled' between batches, exits.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from PIL import Image
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImageRecord, LibraryAuditRun
|
||||
from ..services.audits import single_color, transparency
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BATCH = 500
|
||||
_PROGRESS_TICK = 100
|
||||
_MAX_MATCHED = 50_000
|
||||
|
||||
_RULES = {
|
||||
"transparency": transparency.evaluate,
|
||||
"single_color": single_color.evaluate,
|
||||
}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.library_audit.scan_library_for_rule",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=7200,
|
||||
time_limit=7500,
|
||||
)
|
||||
def scan_library_for_rule(self, audit_id: int) -> dict:
|
||||
"""See module docstring. Returns a small summary dict for eager-mode
|
||||
test assertions (real workers ignore the return value)."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
try:
|
||||
with SessionLocal() as session:
|
||||
audit = session.get(LibraryAuditRun, audit_id)
|
||||
if audit is None:
|
||||
return {"audit_id": audit_id, "status": "missing"}
|
||||
evaluate = _RULES.get(audit.rule)
|
||||
if evaluate is None:
|
||||
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
||||
return {"audit_id": audit_id, "status": "error"}
|
||||
params = dict(audit.params or {})
|
||||
matched: list[int] = []
|
||||
scanned = 0
|
||||
last_id = 0
|
||||
while True:
|
||||
# Cancellation check between batches.
|
||||
current_status = session.execute(
|
||||
select(LibraryAuditRun.status)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
if current_status == "cancelled":
|
||||
return {"audit_id": audit_id, "status": "cancelled"}
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.where(ImageRecord.mime.like("image/%"))
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(_BATCH)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
for image_id, image_path in rows:
|
||||
last_id = image_id
|
||||
scanned += 1
|
||||
try:
|
||||
with Image.open(image_path) as im:
|
||||
try:
|
||||
if evaluate(im, **params):
|
||||
matched.append(image_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"audit %s: rule evaluate failed on %s: %s",
|
||||
audit_id, image_path, exc,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.warning(
|
||||
"audit %s: image_record %s file missing at %s; skipping",
|
||||
audit_id, image_id, image_path,
|
||||
)
|
||||
except OSError as exc:
|
||||
log.warning(
|
||||
"audit %s: PIL load failed for %s: %s",
|
||||
audit_id, image_path, exc,
|
||||
)
|
||||
if len(matched) > _MAX_MATCHED:
|
||||
_mark_error(
|
||||
session, audit_id,
|
||||
f"matched > {_MAX_MATCHED} images; "
|
||||
"tighten threshold and re-run",
|
||||
)
|
||||
return {"audit_id": audit_id, "status": "error"}
|
||||
if scanned % _PROGRESS_TICK == 0:
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(scanned_count=scanned)
|
||||
)
|
||||
session.commit()
|
||||
# Final state.
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(
|
||||
scanned_count=scanned,
|
||||
matched_count=len(matched),
|
||||
matched_ids=matched,
|
||||
status="ready",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return {
|
||||
"audit_id": audit_id,
|
||||
"status": "ready",
|
||||
"scanned": scanned,
|
||||
"matched": len(matched),
|
||||
}
|
||||
except SoftTimeLimitExceeded:
|
||||
with SessionLocal() as session:
|
||||
_mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)")
|
||||
raise
|
||||
except (OperationalError, DBAPIError):
|
||||
# Retryable per the decorator; leave row in 'running' and let
|
||||
# autoretry try again. Recovery sweep catches if all retries fail.
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
tb = traceback.format_exc()
|
||||
with SessionLocal() as session:
|
||||
_mark_error(session, audit_id, tb)
|
||||
raise
|
||||
|
||||
|
||||
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(
|
||||
status="error",
|
||||
error=error_msg,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
@@ -1,12 +1,13 @@
|
||||
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy import and_, delete, or_, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
||||
@@ -16,6 +17,22 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 5
|
||||
# Archive ImportTasks run the per-member pipeline inline for every
|
||||
# member (import_archive_file: soft=30min/hard=35min). The ImportTask
|
||||
# 'processing' recovery sweep must give them a longer threshold or it
|
||||
# re-queues a legitimately-running archive mid-import (double-process).
|
||||
# 40 min = 5-min buffer past the archive task's hard kill.
|
||||
# Operator-flagged 2026-05-28 (target 1645019, a big archive).
|
||||
ARCHIVE_STUCK_THRESHOLD_MINUTES = 40
|
||||
|
||||
# Poison-pill cap. After being recovered (re-queued from a stuck
|
||||
# 'processing' state) MAX_RECOVERY_ATTEMPTS-1 times, the next sweep
|
||||
# marks the row 'failed' instead of looping. 3 = two recoveries then
|
||||
# give up. A row reaches this only if it leaves NO terminal flip each
|
||||
# run — i.e. it hard-crashes the worker (OOM/segfault/SIGKILL), the
|
||||
# signature of a corrupt or oversized input. Caught exceptions already
|
||||
# flip to terminal 'failed' and never enter this loop.
|
||||
MAX_RECOVERY_ATTEMPTS = 3
|
||||
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
@@ -24,17 +41,40 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
|
||||
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
||||
# Tasks/queues that legitimately run longer than the default 5-min
|
||||
# threshold need their own larger value, else the sweep marks in-flight
|
||||
# work 'error' before it finishes. Each value MUST be ≥ the relevant
|
||||
# task.time_limit + a small buffer. task_name overrides take precedence
|
||||
# over queue overrides.
|
||||
#
|
||||
# ml queue: tag_and_embed video branch (≈20 GPU ops); time_limit=1200.
|
||||
# import_archive_file: shares the 'import' queue with the fast
|
||||
# single-file import_media_file, so it needs a task-name override
|
||||
# (the import queue itself stays at the 5-min default for single
|
||||
# files); time_limit=2100.
|
||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"ml": 25,
|
||||
}
|
||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||
}
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||
|
||||
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
|
||||
.delay() and let the import retry. Was 30 min historically;
|
||||
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
|
||||
import_media_file is sub-second for the vast majority of files and
|
||||
capped at the per-task soft_time_limit (5 min), so anything still
|
||||
'processing' after that window is a confirmed crash.
|
||||
1. 'processing' too long — worker crash mid-import. Re-queue via
|
||||
enqueue_import (routing media vs archive) and let the import
|
||||
retry. Threshold is task-type-aware: media files are sub-second
|
||||
and capped at the 5-min soft limit, so STUCK_THRESHOLD_MINUTES
|
||||
(5) means a confirmed crash; archives run the per-member
|
||||
pipeline inline (import_archive_file, 35-min hard limit) so they
|
||||
get ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid re-queueing a
|
||||
still-running archive. (Media was tightened from 30 min to 5
|
||||
2026-05-24 after a 2224-row zombie pile; archive split out
|
||||
2026-05-28.)
|
||||
|
||||
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||
creates rows with status='pending' (commit), then in a second pass
|
||||
@@ -51,53 +91,122 @@ def recover_interrupted_tasks() -> int:
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
media_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
archive_cutoff = now - timedelta(minutes=ARCHIVE_STUCK_THRESHOLD_MINUTES)
|
||||
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
stuck_ids = session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.status == "processing")
|
||||
.where(ImportTask.started_at < processing_cutoff)
|
||||
).scalars().all()
|
||||
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
||||
# blew past psycopg's 65535-parameter ceiling once a sweep covered
|
||||
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
||||
# /import deep scan piled up orphans). Folding the SELECT into the
|
||||
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
||||
# exactly the (id, task_type) pairs that flipped so the requeue
|
||||
# can route media vs archive correctly.
|
||||
#
|
||||
# Media + archive get separate cutoffs: a single media file is
|
||||
# sub-second so 5 min means crash; an archive runs the per-member
|
||||
# pipeline inline and can legitimately take up to its 35-min hard
|
||||
# limit, so it gets ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid
|
||||
# re-queueing a still-running archive.
|
||||
stuck_predicate = and_(
|
||||
ImportTask.status == "processing",
|
||||
or_(
|
||||
and_(ImportTask.task_type != "archive",
|
||||
ImportTask.started_at < media_cutoff),
|
||||
and_(ImportTask.task_type == "archive",
|
||||
ImportTask.started_at < archive_cutoff),
|
||||
),
|
||||
)
|
||||
|
||||
orphan_ids = session.execute(
|
||||
select(ImportTask.id)
|
||||
# POISON-PILL CIRCUIT BREAKER (Layer 1, 2026-05-28). A row that
|
||||
# leaves no terminal flip (hard worker crash: OOM/segfault/SIGKILL
|
||||
# on a corrupt or oversized input) gets re-queued by this sweep —
|
||||
# and would loop forever, re-crashing the worker each pass,
|
||||
# without a cap. Once a row has already been recovered
|
||||
# MAX_RECOVERY_ATTEMPTS-1 times, stop re-queueing it and mark it
|
||||
# 'failed' with a diagnostic so the operator can find + replace
|
||||
# the offending file. This UPDATE runs FIRST so the rows it
|
||||
# claims drop out of 'processing' before the re-queue pass.
|
||||
poison_result = session.execute(
|
||||
update(ImportTask)
|
||||
.where(stuck_predicate)
|
||||
.where(ImportTask.recovery_count >= MAX_RECOVERY_ATTEMPTS - 1)
|
||||
.values(
|
||||
status="failed",
|
||||
finished_at=now,
|
||||
error=(
|
||||
f"crashed or stalled the worker {MAX_RECOVERY_ATTEMPTS} "
|
||||
f"times without completing — likely a corrupt or "
|
||||
f"oversized input. Not re-queued. Inspect/replace the "
|
||||
f"file, then retry via /api/import/retry-failed."
|
||||
),
|
||||
)
|
||||
.returning(ImportTask.id)
|
||||
)
|
||||
poison_ids = [r[0] for r in poison_result.all()]
|
||||
|
||||
# Re-queue the remaining stuck rows (under the cap) and bump
|
||||
# their recovery_count. RETURNING (id, task_type) so the requeue
|
||||
# routes media vs archive correctly.
|
||||
stuck_result = session.execute(
|
||||
update(ImportTask)
|
||||
.where(stuck_predicate)
|
||||
.where(ImportTask.recovery_count < MAX_RECOVERY_ATTEMPTS - 1)
|
||||
.values(
|
||||
status="queued",
|
||||
started_at=None,
|
||||
recovery_count=ImportTask.recovery_count + 1,
|
||||
error="recovered from stuck state",
|
||||
)
|
||||
.returning(ImportTask.id, ImportTask.task_type)
|
||||
)
|
||||
stuck = stuck_result.all()
|
||||
|
||||
orphan_result = session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.status.in_(["pending", "queued"]))
|
||||
.where(ImportTask.created_at < orphan_cutoff)
|
||||
).scalars().all()
|
||||
|
||||
if not stuck_ids and not orphan_ids:
|
||||
return 0
|
||||
|
||||
if stuck_ids:
|
||||
session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(stuck_ids))
|
||||
.values(status="queued", started_at=None, error="recovered from stuck state")
|
||||
)
|
||||
|
||||
if orphan_ids:
|
||||
session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(orphan_ids))
|
||||
.values(
|
||||
status="failed",
|
||||
error=(
|
||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||
"(scanner likely crashed mid-enqueue); retry via "
|
||||
"/api/import/retry-failed"
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="failed",
|
||||
error=(
|
||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||
"(scanner likely crashed mid-enqueue); retry via "
|
||||
"/api/import/retry-failed"
|
||||
),
|
||||
)
|
||||
)
|
||||
orphan_count = orphan_result.rowcount or 0
|
||||
|
||||
session.commit()
|
||||
|
||||
if stuck_ids:
|
||||
from .import_file import import_media_file
|
||||
for tid in stuck_ids:
|
||||
import_media_file.delay(tid)
|
||||
if stuck:
|
||||
from .import_file import enqueue_import
|
||||
for tid, task_type in stuck:
|
||||
enqueue_import(tid, task_type)
|
||||
|
||||
return len(stuck_ids) + len(orphan_ids)
|
||||
# Layer-2 auto re-download (env-gated, default OFF). For each
|
||||
# poison-pill row that resolves to a pollable Source, delete the
|
||||
# bad file and trigger ONE source re-check to fetch a fresh
|
||||
# copy. Bounded by ImportTask.refetched so source-side
|
||||
# corruption can't loop. The 'failed' row stays as history; the
|
||||
# re-downloaded file re-imports as a fresh task on the next scan.
|
||||
if poison_ids and os.environ.get("FC_AUTO_REFETCH_CORRUPT", "0") == "1":
|
||||
from ..models import ImportSettings
|
||||
from ..services.refetch_service import attempt_refetch
|
||||
import_root = Path(session.execute(
|
||||
select(ImportSettings.import_scan_path)
|
||||
.where(ImportSettings.id == 1)
|
||||
).scalar_one())
|
||||
for pid in poison_ids:
|
||||
ptask = session.get(ImportTask, pid)
|
||||
if ptask is None:
|
||||
continue
|
||||
try:
|
||||
attempt_refetch(session, ptask, import_root)
|
||||
except Exception as exc: # noqa: BLE001 — best-effort
|
||||
log.warning("auto-refetch failed for task %s: %s", pid, exc)
|
||||
|
||||
return len(stuck) + len(poison_ids) + orphan_count
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||
@@ -122,18 +231,29 @@ def cleanup_old_tasks() -> int:
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||
def recover_stalled_task_runs() -> int:
|
||||
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
||||
to 'error'. FC-3i.
|
||||
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||
threshold to 'error'. FC-3i.
|
||||
|
||||
A row gets stuck when the worker dies without emitting
|
||||
task_postrun / task_failure (e.g. OOM, container restart between
|
||||
signals, signal handler raised+logged). Shares the 5-min threshold
|
||||
with recover_interrupted_tasks for consistency.
|
||||
signals, signal handler raised+logged). The default 5-min threshold
|
||||
fits short-lived queues (import/thumbnail/download); queues that
|
||||
legitimately run longer tasks (ml-video, deep scans) get their
|
||||
own larger threshold via QUEUE_STUCK_THRESHOLD_MINUTES so the
|
||||
sweep doesn't preempt them.
|
||||
|
||||
Runs once per distinct threshold value: each pass updates rows
|
||||
whose queue maps to that threshold.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
now = datetime.now(UTC)
|
||||
override_tasks = set(TASK_STUCK_THRESHOLD_MINUTES.keys())
|
||||
override_queues = set(QUEUE_STUCK_THRESHOLD_MINUTES.keys())
|
||||
total = 0
|
||||
|
||||
def _flag(minutes, *extra_where):
|
||||
cutoff = now - timedelta(minutes=minutes)
|
||||
stmt = (
|
||||
update(TaskRun)
|
||||
.where(TaskRun.status == "running")
|
||||
.where(TaskRun.started_at < cutoff)
|
||||
@@ -141,14 +261,42 @@ def recover_stalled_task_runs() -> int:
|
||||
status="error",
|
||||
error_type="RecoverySweep",
|
||||
error_message=(
|
||||
f"no completion signal received within "
|
||||
f"{STUCK_THRESHOLD_MINUTES} min"
|
||||
f"no completion signal received within {minutes} min"
|
||||
),
|
||||
finished_at=datetime.now(UTC),
|
||||
finished_at=now,
|
||||
)
|
||||
)
|
||||
for w in extra_where:
|
||||
stmt = stmt.where(w)
|
||||
return session.execute(stmt).rowcount or 0
|
||||
|
||||
with SessionLocal() as session:
|
||||
# Precedence: task_name override → queue override → default.
|
||||
# Each pass excludes rows claimed by a higher-precedence pass so
|
||||
# every row is touched at most once.
|
||||
|
||||
# 1. Per-task-name overrides (e.g. import_archive_file, which
|
||||
# shares the 'import' queue with fast single-file imports).
|
||||
for task_name, minutes in TASK_STUCK_THRESHOLD_MINUTES.items():
|
||||
total += _flag(minutes, TaskRun.task_name == task_name)
|
||||
|
||||
# 2. Per-queue overrides, excluding the override task-names.
|
||||
for queue, minutes in QUEUE_STUCK_THRESHOLD_MINUTES.items():
|
||||
wheres = [TaskRun.queue == queue]
|
||||
if override_tasks:
|
||||
wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||
total += _flag(minutes, *wheres)
|
||||
|
||||
# 3. Default — everything not claimed above.
|
||||
default_wheres = []
|
||||
if override_queues:
|
||||
default_wheres.append(TaskRun.queue.notin_(override_queues))
|
||||
if override_tasks:
|
||||
default_wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||
total += _flag(STUCK_THRESHOLD_MINUTES, *default_wheres)
|
||||
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
return total
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||
|
||||
+21
-2
@@ -31,8 +31,15 @@ def _is_video(path: Path) -> bool:
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=300,
|
||||
time_limit=420,
|
||||
# Sized for the video branch: sample 10 frames, run tagger +
|
||||
# embedder on each (≈20 GPU ops vs 2 for an image). A loaded
|
||||
# ml-worker can take 5-10 min on a long video; bumped from
|
||||
# 5min/7min on 2026-05-28 after operator-flagged image 6288 (a
|
||||
# .mp4) hit the recovery sweep at 5 min while still legitimately
|
||||
# processing. Image runs return in seconds; the bump doesn't
|
||||
# affect their UX.
|
||||
soft_time_limit=900, # 15 min
|
||||
time_limit=1200, # 20 min hard
|
||||
)
|
||||
def tag_and_embed(self, image_id: int) -> dict:
|
||||
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
||||
@@ -64,6 +71,18 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
embedder = get_embedder()
|
||||
|
||||
if _is_video(src):
|
||||
# Layer-3 isolation: ffprobe (a separate process) validates
|
||||
# the container before we burn ~20 GPU ops sampling frames
|
||||
# from it. A corrupt video that would crash the frame
|
||||
# decoder is rejected cleanly here instead of taking down
|
||||
# the ml-worker. Operator-flagged 2026-05-28.
|
||||
from ..utils import safe_probe
|
||||
vprobe = safe_probe.probe_video(src)
|
||||
if not vprobe.ok:
|
||||
return {
|
||||
"status": "bad_video", "image_id": image_id,
|
||||
"reason": vprobe.reason,
|
||||
}
|
||||
frames = _sample_video_frames(
|
||||
src, int(os.environ.get("VIDEO_ML_FRAMES", "10"))
|
||||
)
|
||||
|
||||
+26
-13
@@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
||||
from ..services.archive_extractor import is_archive
|
||||
from ..services.scheduler_service import select_due_sources
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
@@ -59,16 +60,25 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
session.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
# Skip-set: any source_path that already has a non-failed ImportTask
|
||||
# row. Re-running scan_directory must not re-enqueue files the
|
||||
# importer has already handled (or is currently handling); doing so
|
||||
# creates duplicate work and inflates the queue. Failed prior tasks
|
||||
# are eligible for retry.
|
||||
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
|
||||
#
|
||||
# quick: any non-failed prior ImportTask (active OR finished) is
|
||||
# skipped — quick scan only does new-file enqueue, so re-touching
|
||||
# already-imported files is wasted work.
|
||||
#
|
||||
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
|
||||
# are skipped. Completed and skipped tasks ARE re-queued because
|
||||
# deep scan exists precisely to re-touch already-imported files
|
||||
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
|
||||
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
|
||||
active_statuses = ["pending", "queued", "processing"]
|
||||
if mode == "deep":
|
||||
skip_statuses = active_statuses
|
||||
else:
|
||||
skip_statuses = active_statuses + ["complete", "skipped"]
|
||||
non_failed_existing = set(session.execute(
|
||||
select(ImportTask.source_path).where(
|
||||
ImportTask.status.in_(
|
||||
["pending", "queued", "processing", "complete", "skipped"]
|
||||
),
|
||||
ImportTask.status.in_(skip_statuses),
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
@@ -87,7 +97,9 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
task = ImportTask(
|
||||
batch_id=batch_id,
|
||||
source_path=entry_str,
|
||||
task_type="media",
|
||||
# Archives route to import_archive_file (larger time
|
||||
# budget) — they run the per-member pipeline inline.
|
||||
task_type="archive" if is_archive(entry) else "media",
|
||||
status="pending",
|
||||
size_bytes=size,
|
||||
)
|
||||
@@ -106,15 +118,16 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
batch.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
|
||||
# Now enqueue import_media_file for each pending task.
|
||||
# Now enqueue each pending task on the right Celery task
|
||||
# (media vs archive) via the shared router.
|
||||
from .import_file import enqueue_import
|
||||
|
||||
for task in session.execute(
|
||||
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
||||
).scalars():
|
||||
task.status = "queued"
|
||||
session.add(task)
|
||||
from .import_file import import_media_file
|
||||
|
||||
import_media_file.delay(task.id)
|
||||
enqueue_import(task.id, task.task_type)
|
||||
session.commit()
|
||||
|
||||
if mode == "deep":
|
||||
|
||||
@@ -17,6 +17,30 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
||||
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def _thumb_is_valid(path: Path) -> bool:
|
||||
"""Return True iff `path` exists and starts with a JPEG or PNG magic header.
|
||||
|
||||
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for
|
||||
opaque sources, PNG for alpha sources. Anything else (missing file, OSError,
|
||||
truncated, wrong magic) is invalid.
|
||||
"""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
head = f.read(12)
|
||||
except OSError:
|
||||
return False
|
||||
if len(head) < 8:
|
||||
return False
|
||||
if head[:3] == THUMB_MAGIC_JPEG:
|
||||
return True
|
||||
if head[:8] == THUMB_MAGIC_PNG:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
||||
@@ -50,3 +74,58 @@ def generate_thumbnail(self, image_id: int) -> dict:
|
||||
session.add(record)
|
||||
session.commit()
|
||||
return {"status": "ok", "image_id": image_id, "path": str(result.path)}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.thumbnail.backfill_thumbnails",
|
||||
bind=True,
|
||||
)
|
||||
def backfill_thumbnails(self) -> dict:
|
||||
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
||||
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
||||
|
||||
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for
|
||||
rows that point at a missing or corrupt file before enqueueing — keeps
|
||||
the DB self-consistent on partial runs and makes re-runs safe.
|
||||
|
||||
Returns {"enqueued": N, "ok": M, "regenerated": K} where:
|
||||
- enqueued = total generate_thumbnail.delay() calls
|
||||
- ok = rows whose existing thumbnail file is valid (skipped)
|
||||
- regenerated = subset of enqueued that had a non-NULL thumbnail_path
|
||||
cleared (i.e. missing + corrupt)
|
||||
"""
|
||||
from sqlalchemy import select, update
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
enqueued = 0
|
||||
ok = 0
|
||||
regenerated = 0
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(500)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
for image_id, thumb_path in rows:
|
||||
if thumb_path is None:
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
elif _thumb_is_valid(Path(thumb_path)):
|
||||
ok += 1
|
||||
else:
|
||||
session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.id == image_id)
|
||||
.values(thumbnail_path=None)
|
||||
)
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
regenerated += 1
|
||||
session.commit()
|
||||
last_id = rows[-1][0]
|
||||
return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated}
|
||||
|
||||
@@ -13,8 +13,21 @@ HASH_SIZE = 8
|
||||
|
||||
def compute_phash(pil_image) -> str | None:
|
||||
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
||||
failure (videos/unreadable/non-image)."""
|
||||
failure (videos/unreadable/non-image).
|
||||
|
||||
For animated images (multi-frame WebP/GIF/APNG), explicitly seek to
|
||||
frame 0 first. Without this, some PIL operations downstream of
|
||||
imagehash.phash (convert("L"), resize) can iterate all frames and
|
||||
blow past Celery's hard time limit on large animations
|
||||
(operator-flagged 2026-05-26 against animated WebPs). The pHash of
|
||||
frame 0 is the conventional choice for animated content.
|
||||
"""
|
||||
try:
|
||||
if getattr(pil_image, "is_animated", False):
|
||||
try:
|
||||
pil_image.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Subprocess-isolated media probes (Layer 3 of import resilience).
|
||||
|
||||
A malformed video or archive can hard-crash the worker process — a
|
||||
decoder OOM, a native-lib segfault, or a decompression bomb. A hard
|
||||
crash leaves no terminal flip, so the recovery sweep re-queues the row
|
||||
and it crashes again: a poison-pill loop (the Layer-1 cap is the
|
||||
backstop, but isolating the crash is better — the file gets a clean
|
||||
terminal failure and the worker never dies).
|
||||
|
||||
These probes run the risky read in a way that contains the blast:
|
||||
|
||||
- Video: `ffprobe` is a separate binary, so a crash decoding the
|
||||
container kills only ffprobe (non-zero exit), never the worker. Also
|
||||
returns width/height, which the importer didn't previously capture
|
||||
for videos.
|
||||
- Archive: an uncompressed-size guard (catches decompression bombs
|
||||
before they OOM anything) plus an integrity test in a spawned child
|
||||
(catches native-lib crashes on a malformed archive). A child segfault
|
||||
/ OOM shows up as a non-zero exit code, not a dead worker.
|
||||
|
||||
Images are intentionally NOT probed here: Pillow raises (it doesn't
|
||||
segfault) on the realistic corrupt-image cases, the importer already
|
||||
catches that as an invalid_image skip, and a subprocess per image would
|
||||
wreck deep-scan throughput on a large library. Add an image branch only
|
||||
if a real image-induced worker crash is ever observed.
|
||||
|
||||
Operator-requested 2026-05-28 (Layer 3).
|
||||
"""
|
||||
|
||||
import json
|
||||
import multiprocessing as mp
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
VIDEO_PROBE_TIMEOUT_SECONDS = 60
|
||||
ARCHIVE_PROBE_TIMEOUT_SECONDS = 120
|
||||
# Refuse archives whose total UNCOMPRESSED size exceeds this — the
|
||||
# classic decompression-bomb guard (a 4 GB cap comfortably clears real
|
||||
# art-pack archives while stopping a few-KB zip that expands to TB).
|
||||
MAX_ARCHIVE_UNCOMPRESSED_BYTES = 4 * 1024 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeResult:
|
||||
ok: bool
|
||||
# crashed=True means the probe HARD-FAILED (subprocess killed by a
|
||||
# signal, OOM, or timeout) — the poison-pill signature. crashed=False
|
||||
# with ok=False means a clean rejection (corrupt-but-handled,
|
||||
# bomb-size-exceeded, integrity mismatch). Callers map crashed → a
|
||||
# terminal 'failed', clean → a 'skipped'/'failed' of their choosing.
|
||||
crashed: bool = False
|
||||
reason: str | None = None
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
|
||||
|
||||
def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||
"""Validate a video container + first video stream via ffprobe."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=width,height",
|
||||
"-of", "json", str(path),
|
||||
],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return ProbeResult(ok=False, crashed=True, reason="ffprobe timed out")
|
||||
except OSError as exc:
|
||||
# ffprobe missing / not executable — environmental, not the
|
||||
# file's fault. Treat as a clean non-crash failure so the import
|
||||
# path can decide (it currently proceeds without dims).
|
||||
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe unavailable: {exc}")
|
||||
if out.returncode != 0:
|
||||
return ProbeResult(
|
||||
ok=False, crashed=False,
|
||||
reason=f"ffprobe rejected the file: {out.stderr.strip()[:200]}",
|
||||
)
|
||||
try:
|
||||
streams = (json.loads(out.stdout) or {}).get("streams") or []
|
||||
except json.JSONDecodeError as exc:
|
||||
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe output parse failed: {exc}")
|
||||
if not streams:
|
||||
return ProbeResult(ok=False, crashed=False, reason="no decodable video stream")
|
||||
return ProbeResult(
|
||||
ok=True, width=streams[0].get("width"), height=streams[0].get("height"),
|
||||
)
|
||||
|
||||
|
||||
def probe_archive(path: Path, *, timeout: float = ARCHIVE_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||
"""Bomb-size guard + isolated integrity test for an archive."""
|
||||
ctx = mp.get_context("spawn")
|
||||
q = ctx.Queue()
|
||||
proc = ctx.Process(target=_archive_probe_target, args=(str(path), q))
|
||||
proc.start()
|
||||
proc.join(timeout)
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
proc.join(5)
|
||||
return ProbeResult(ok=False, crashed=True, reason="archive probe timed out")
|
||||
if proc.exitcode != 0:
|
||||
# Negative exitcode = killed by signal (segfault); positive =
|
||||
# the child os._exit'd or was OOM-killed. Either way the file
|
||||
# hard-crashed the probe — the poison-pill signature.
|
||||
return ProbeResult(
|
||||
ok=False, crashed=True,
|
||||
reason=f"archive probe crashed (exit {proc.exitcode})",
|
||||
)
|
||||
try:
|
||||
outcome = q.get(timeout=5)
|
||||
except Exception: # noqa: BLE001 — empty queue / broken pipe
|
||||
return ProbeResult(ok=False, crashed=True, reason="archive probe produced no result")
|
||||
status, detail = outcome
|
||||
if status == "ok":
|
||||
return ProbeResult(ok=True)
|
||||
return ProbeResult(ok=False, crashed=False, reason=detail)
|
||||
|
||||
|
||||
def _archive_probe_target(path_str: str, q) -> None:
|
||||
"""Runs in the spawned child. Reads member sizes (bomb guard) then
|
||||
runs the format's integrity test. Puts ('ok', None) or
|
||||
('error', reason). A crash/OOM here never reaches the queue — the
|
||||
parent reads the non-zero exit code instead."""
|
||||
path = Path(path_str)
|
||||
ext = path.suffix.lower()
|
||||
try:
|
||||
total, test_bad = _inspect_archive(path, ext)
|
||||
except Exception as exc: # noqa: BLE001 — clean rejection
|
||||
q.put(("error", f"{type(exc).__name__}: {exc}"))
|
||||
return
|
||||
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||
gib = total / (1024 ** 3)
|
||||
q.put(("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap"))
|
||||
return
|
||||
if test_bad is not None:
|
||||
q.put(("error", f"integrity test failed at member {test_bad!r}"))
|
||||
return
|
||||
q.put(("ok", None))
|
||||
|
||||
|
||||
def _inspect_archive(path: Path, ext: str):
|
||||
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||
for the archive. Format-specific; raises on a structurally-broken
|
||||
container (caught by the child as a clean rejection)."""
|
||||
if ext in (".zip", ".cbz"):
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
total = sum(zi.file_size for zi in zf.infolist())
|
||||
return total, zf.testzip()
|
||||
if ext == ".rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||
rf.testrar()
|
||||
return total, None
|
||||
if ext == ".7z":
|
||||
import py7zr
|
||||
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
info = zf.archiveinfo()
|
||||
total = getattr(info, "uncompressed", None)
|
||||
ok = zf.test() # True / None when all members pass
|
||||
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||
# Unknown extension — nothing to test; treat as clean.
|
||||
return None, None
|
||||
@@ -1,13 +1,22 @@
|
||||
"""Minimal gallery-dl sidecar parsing (one-time filesystem-import aid).
|
||||
|
||||
No per-platform branching: a small common key set with fallbacks; the
|
||||
full JSON is kept in raw so anything unmapped is recoverable later.
|
||||
Per-platform quirks (post_url synthesis, key-chain overrides) live in
|
||||
the platforms registry — `backend/app/services/platforms/`. This module
|
||||
is platform-agnostic: it looks up `category` in the sidecar and asks
|
||||
the registry for the right behavior.
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.platforms import (
|
||||
PLATFORMS,
|
||||
description_keys_for,
|
||||
external_post_id_keys_for,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SidecarData:
|
||||
@@ -21,10 +30,28 @@ class SidecarData:
|
||||
raw: dict
|
||||
|
||||
|
||||
# gallery-dl prefixes media filenames with `NN_` for in-post ordering
|
||||
# (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar
|
||||
# under the attachment's stem WITHOUT that ordering prefix
|
||||
# (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars.
|
||||
# Confirmed against real Patreon downloads 2026-05-26 — without this
|
||||
# strip, every gallery-dl post-level sidecar was invisible to FC since
|
||||
# FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB).
|
||||
_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
|
||||
|
||||
|
||||
def find_sidecar(media: Path) -> Path | None:
|
||||
# Attachment-level sidecars (image.jpg.json, image.json).
|
||||
for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
|
||||
if cand.is_file():
|
||||
return cand
|
||||
# gallery-dl post-numbered convention: strip the `NN_` prefix from
|
||||
# the stem and look for that.json in the same directory.
|
||||
m = _NUMBERING_PREFIX.match(media.stem)
|
||||
if m:
|
||||
cand = media.parent / f"{m.group(1)}.json"
|
||||
if cand.is_file():
|
||||
return cand
|
||||
return None
|
||||
|
||||
|
||||
@@ -36,6 +63,46 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_id(data: dict, keys: tuple[str, ...]) -> str | None:
|
||||
"""Like `_first_str` but accepts ints and rejects bool (Python's
|
||||
bool subclasses int, so a literal `"id": true` would otherwise
|
||||
yield external_post_id="True")."""
|
||||
for k in keys:
|
||||
v = data.get(k)
|
||||
if isinstance(v, bool):
|
||||
continue
|
||||
if isinstance(v, (str, int)) and str(v).strip():
|
||||
return str(v).strip()
|
||||
return None
|
||||
|
||||
|
||||
# Strip HTML tags + collapse whitespace + take the first non-empty line.
|
||||
# Used to derive a display title from a body when the platform doesn't
|
||||
# expose a separate title field (subscribestar posts always write
|
||||
# `title: ""` and put the leading sentence inside `content` as HTML).
|
||||
# Truncated to 120 chars with an ellipsis if longer — long enough to be
|
||||
# meaningful in a feed, short enough to fit a row.
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _first_line_text(body: str, limit: int = 120) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
text = _TAG_RE.sub(" ", body)
|
||||
text = text.replace("\xa0", " ")
|
||||
# Split on hard line breaks first; the body-stripped HTML often
|
||||
# collapses to one logical line, in which case the first sentence
|
||||
# split is the next-best heuristic.
|
||||
for line in text.splitlines():
|
||||
line = _WS_RE.sub(" ", line).strip()
|
||||
if line:
|
||||
if len(line) > limit:
|
||||
return line[: limit - 1].rstrip() + "…"
|
||||
return line
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date(v) -> datetime | None:
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
@@ -65,14 +132,7 @@ def parse_sidecar(data: dict) -> SidecarData:
|
||||
cat = data.get("category")
|
||||
platform = cat if isinstance(cat, str) and cat.strip() else None
|
||||
|
||||
external_post_id = None
|
||||
for k in ("id", "post_id", "index", "message_id"):
|
||||
v = data.get(k)
|
||||
if isinstance(v, bool):
|
||||
continue
|
||||
if isinstance(v, (str, int)) and str(v).strip():
|
||||
external_post_id = str(v)
|
||||
break
|
||||
external_post_id = _first_id(data, external_post_id_keys_for(platform))
|
||||
|
||||
pc = data.get("page_count")
|
||||
if isinstance(pc, bool):
|
||||
@@ -92,12 +152,32 @@ def parse_sidecar(data: dict) -> SidecarData:
|
||||
if post_date is not None:
|
||||
break
|
||||
|
||||
description = _first_str(data, description_keys_for(platform))
|
||||
|
||||
# When `title` is empty (subscribestar always; sometimes elsewhere),
|
||||
# synthesize from the description body's first non-empty text line.
|
||||
# Patreon's explicit titles short-circuit the fallback.
|
||||
post_title = _first_str(data, ("title",))
|
||||
if post_title is None and description:
|
||||
post_title = _first_line_text(description)
|
||||
|
||||
# post_url: ask the platform module to synthesize a permalink.
|
||||
# When the platform registers a `derive_post_url`, it owns the
|
||||
# field (the bare `url`/`post_url` value is a file CDN URL and
|
||||
# must NEVER be persisted). When it doesn't register one, trust
|
||||
# the sidecar's `url` (Patreon's case — real permalink).
|
||||
info = PLATFORMS.get(platform) if platform else None
|
||||
if info is not None and info.derive_post_url is not None:
|
||||
post_url = info.derive_post_url(data)
|
||||
else:
|
||||
post_url = _first_str(data, ("url", "post_url"))
|
||||
|
||||
return SidecarData(
|
||||
platform=platform,
|
||||
external_post_id=external_post_id,
|
||||
post_url=_first_str(data, ("url", "post_url")),
|
||||
post_title=_first_str(data, ("title",)),
|
||||
description=_first_str(data, ("content", "description", "caption")),
|
||||
post_url=post_url,
|
||||
post_title=post_title,
|
||||
description=description,
|
||||
attachment_count=attachment_count,
|
||||
post_date=post_date,
|
||||
raw=data,
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Parse the user-facing `kind:name` shortcut used by the add-tag input.
|
||||
|
||||
Mirrors IR's app/utils/tag_prefix.py. Tag.name in FC is stored bare;
|
||||
the `kind:` prefix only exists as an input convention at user-facing
|
||||
places (image-modal add-tag input, future bulk-add forms). The parser
|
||||
is the single owner of the kind-string list — anything not in
|
||||
KNOWN_KINDS keeps its colon as literal text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Kinds the user can type as a prefix at the input boundary.
|
||||
# Exclusions:
|
||||
# - `general` is the default for un-prefixed input (never typed as prefix)
|
||||
# - `archive`, `post` are system-managed
|
||||
# - `artist` was retired in FC-2d-vii-c — artists are first-class
|
||||
# entities (Artist row + ImageRecord.artist_id), browsed via the
|
||||
# provenance axis rather than as tags. See project_provenance_separation.
|
||||
# - `meta`, `rating` retired as user-typeable per operator 2026-05-26 —
|
||||
# content classification only needs character/fandom/series.
|
||||
KNOWN_KINDS: frozenset[str] = frozenset({
|
||||
"character",
|
||||
"fandom",
|
||||
"series",
|
||||
})
|
||||
|
||||
|
||||
def parse_kind_prefix(raw: str) -> tuple[str | None, str]:
|
||||
"""Split a raw user-typed tag string into (kind, name).
|
||||
|
||||
Returns (kind, name) where kind is lowercase canonical and in
|
||||
KNOWN_KINDS, or (None, raw.strip()) if no recognized prefix is
|
||||
present. `name` is always whitespace-stripped.
|
||||
|
||||
Examples:
|
||||
parse_kind_prefix("character:Saber") -> ("character", "Saber")
|
||||
parse_kind_prefix("Character:Saber") -> ("character", "Saber")
|
||||
parse_kind_prefix("sunset") -> (None, "sunset")
|
||||
parse_kind_prefix("http://example") -> (None, "http://example")
|
||||
parse_kind_prefix("fandom: FSN ") -> ("fandom", "FSN")
|
||||
"""
|
||||
if ":" in raw:
|
||||
prefix, rest = raw.split(":", 1)
|
||||
if prefix.lower() in KNOWN_KINDS:
|
||||
return prefix.lower(), rest.strip()
|
||||
return None, raw.strip()
|
||||
@@ -38,11 +38,33 @@ function deduplicateCookies(cookies) {
|
||||
function toNetscapeFormat(cookies) {
|
||||
const lines = ['# Netscape HTTP Cookie File'];
|
||||
for (const c of cookies) {
|
||||
let domain = c.domain.replace(/^\.?www\./, '.');
|
||||
if (!domain.startsWith('.')) domain = '.' + domain;
|
||||
// Preserve the browser's actual scope. Earlier versions rewrote
|
||||
// every cookie to a leading-dot subdomain-wide form, which broke
|
||||
// gallery-dl's HF extractor: its `cookies.get(name,
|
||||
// domain="www.hentai-foundry.com")` does EXACT domain matching and
|
||||
// missed host-only PHPSESSID rewritten to `.hentai-foundry.com`.
|
||||
// Operator-flagged 2026-05-27. Backend `_augment_cookies` covers
|
||||
// the already-stored cookies; this fix is forward-compat for fresh
|
||||
// captures.
|
||||
//
|
||||
// Cookie storage semantics (Firefox):
|
||||
// c.hostOnly === true → cookie set without a Domain= attribute;
|
||||
// applies to the exact host only.
|
||||
// c.hostOnly === false → cookie set with Domain=X; applies to
|
||||
// that domain and its subdomains.
|
||||
//
|
||||
// Netscape format:
|
||||
// leading-dot domain + TRUE flag → subdomain-wide
|
||||
// bare-host domain + FALSE flag → host-only
|
||||
const hostOnly = c.hostOnly === true;
|
||||
let domain = c.domain;
|
||||
if (!hostOnly && !domain.startsWith('.')) {
|
||||
domain = '.' + domain;
|
||||
}
|
||||
const subdomainFlag = hostOnly ? 'FALSE' : 'TRUE';
|
||||
const secure = c.secure ? 'TRUE' : 'FALSE';
|
||||
const expiration = c.expirationDate ? Math.floor(c.expirationDate) : 0;
|
||||
lines.push([domain, 'TRUE', c.path || '/', secure, String(expiration), c.name, c.value].join('\t'));
|
||||
lines.push([domain, subdomainFlag, c.path || '/', secure, String(expiration), c.name, c.value].join('\t'));
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.5",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
@@ -11,6 +11,11 @@
|
||||
}
|
||||
},
|
||||
|
||||
"content_security_policy": {
|
||||
"_comment": "Override the MV3 default CSP to OMIT upgrade-insecure-requests. FC runs over plain HTTP per the homelab posture (feedback_homelab_http), and the default MV3 CSP would silently upgrade every fetch(http://curator.../...) to https:// and fail with NS_ERROR_GENERATE_FAILURE. Operator-flagged 2026-05-26 after the 'Test connection' button errored despite a working CORS preflight on the backend.",
|
||||
"extension_pages": "script-src 'self'; object-src 'self';"
|
||||
},
|
||||
|
||||
"permissions": [
|
||||
"cookies",
|
||||
"storage",
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.5",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
"lint": "web-ext lint --source-dir=.",
|
||||
"start": "web-ext run --source-dir=. --firefox=firefox",
|
||||
"build": "web-ext build --source-dir=. --overwrite-dest",
|
||||
"sign": "web-ext sign --source-dir=. --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
"lint": "web-ext lint --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore",
|
||||
"start": "web-ext run --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --firefox=firefox",
|
||||
"build": "web-ext build --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --overwrite-dest",
|
||||
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
},
|
||||
"devDependencies": {
|
||||
"web-ext": "^8.0.0"
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
module.exports = {
|
||||
sourceDir: '.',
|
||||
artifactsDir: './web-ext-artifacts',
|
||||
ignoreFiles: [
|
||||
'package.json',
|
||||
'package-lock.json',
|
||||
'web-ext-config.cjs',
|
||||
'web-ext-artifacts',
|
||||
'node_modules',
|
||||
'README.md',
|
||||
'.gitignore',
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
<template>
|
||||
<div class="fc-artist-gallery">
|
||||
<MasonryGrid
|
||||
:items="store.images"
|
||||
:loading="store.imagesLoading"
|
||||
:has-more="store.hasMoreImages"
|
||||
@load-more="store.loadMoreImages(props.slug)"
|
||||
@open="openImage"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
|
||||
import { useArtistStore } from '../../stores/artist.js'
|
||||
import { useModalStore } from '../../stores/modal.js'
|
||||
import MasonryGrid from '../discovery/MasonryGrid.vue'
|
||||
|
||||
const props = defineProps({
|
||||
slug: { type: String, required: true },
|
||||
})
|
||||
|
||||
const store = useArtistStore()
|
||||
const modal = useModalStore()
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
onMounted(() => {
|
||||
const initial = parseInt(route.query.image, 10)
|
||||
if (!isNaN(initial)) modal.open(initial)
|
||||
})
|
||||
|
||||
watch(() => route.query.image, (q) => {
|
||||
const id = parseInt(q, 10)
|
||||
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
||||
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
||||
})
|
||||
|
||||
function openImage (id) {
|
||||
router.push({ query: { ...route.query, image: id } })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-artist-gallery { min-width: 0; }
|
||||
</style>
|
||||
@@ -0,0 +1,128 @@
|
||||
<template>
|
||||
<header class="fc-artist-header">
|
||||
<div class="fc-artist-header__left">
|
||||
<h1 class="fc-artist-header__name">{{ name }}</h1>
|
||||
<span v-if="stats" class="fc-artist-header__stats">{{ stats }}</span>
|
||||
</div>
|
||||
<v-tabs
|
||||
:model-value="modelValue"
|
||||
color="accent"
|
||||
density="compact"
|
||||
class="fc-artist-header__tabs"
|
||||
@update:model-value="$emit('update:modelValue', $event)"
|
||||
>
|
||||
<v-tab value="posts">
|
||||
Posts
|
||||
<span v-if="postCount != null" class="fc-artist-header__tab-count">
|
||||
({{ postCount }})
|
||||
</span>
|
||||
</v-tab>
|
||||
<v-tab value="gallery">
|
||||
Gallery
|
||||
<span v-if="imageCount != null" class="fc-artist-header__tab-count">
|
||||
({{ imageCount }})
|
||||
</span>
|
||||
</v-tab>
|
||||
<v-tab value="management">Management</v-tab>
|
||||
</v-tabs>
|
||||
|
||||
<!-- Right-side spacer: balances the left cell's flex weight so the
|
||||
centered tabs stay geometrically centered regardless of the
|
||||
artist-name length. Mirrors TopNav's 1fr | auto | 1fr layout. -->
|
||||
<div class="fc-artist-header__right" />
|
||||
</header>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
|
||||
const props = defineProps({
|
||||
name: { type: String, required: true },
|
||||
imageCount: { type: Number, default: null },
|
||||
postCount: { type: Number, default: null },
|
||||
lastAdded: { type: String, default: null },
|
||||
modelValue: { type: String, required: true },
|
||||
})
|
||||
|
||||
defineEmits(['update:modelValue'])
|
||||
|
||||
const stats = computed(() => {
|
||||
const parts = []
|
||||
if (props.imageCount != null) {
|
||||
parts.push(`${props.imageCount} image${props.imageCount === 1 ? '' : 's'}`)
|
||||
}
|
||||
if (props.lastAdded) {
|
||||
parts.push(`last added ${props.lastAdded.slice(0, 10)}`)
|
||||
}
|
||||
return parts.join(' · ')
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* Matches TopNav.vue's frosted recipe exactly. top:48px parks it flush
|
||||
against TopNav's bottom edge (TopNav is 0.75rem padding + ~24px content
|
||||
= ~48px tall; operator-flagged 2026-05-26 that top:64px left a visible
|
||||
gap). The two bars now read as one continuous frosted strip. */
|
||||
.fc-artist-header {
|
||||
position: sticky;
|
||||
top: 48px;
|
||||
z-index: 4;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(20, 23, 26, 0.92) 0%,
|
||||
rgba(20, 23, 26, 0.65) 60%,
|
||||
rgba(20, 23, 26, 0) 100%
|
||||
);
|
||||
backdrop-filter: blur(2px);
|
||||
-webkit-backdrop-filter: blur(2px);
|
||||
}
|
||||
|
||||
.fc-artist-header__left {
|
||||
flex: 1 1 0;
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.fc-artist-header__name {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 24px;
|
||||
font-weight: 500;
|
||||
margin: 0;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.fc-artist-header__stats {
|
||||
font-size: 13px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.fc-artist-header__tabs {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.fc-artist-header__right {
|
||||
flex: 1 1 0;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.fc-artist-header__tab-count {
|
||||
margin-left: 4px;
|
||||
font-size: 12px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,133 @@
|
||||
<template>
|
||||
<div class="fc-artist-mgmt">
|
||||
<section class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Overview</h2>
|
||||
<div class="fc-artist-mgmt__chips">
|
||||
<v-chip
|
||||
size="small"
|
||||
:variant="overview.is_subscription ? 'flat' : 'outlined'"
|
||||
:color="overview.is_subscription ? 'accent' : undefined"
|
||||
prepend-icon="mdi-rss"
|
||||
>{{ overview.is_subscription ? 'Subscription' : 'One-off' }}</v-chip>
|
||||
<v-chip
|
||||
size="small" variant="outlined" prepend-icon="mdi-link-variant"
|
||||
:to="`/subscriptions?artist_id=${overview.id}`"
|
||||
>{{ overview.sources.length }} subscription{{ overview.sources.length === 1 ? '' : 's' }}</v-chip>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="overview.cooccurring_tags.length" class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Frequent tags</h2>
|
||||
<div class="fc-artist-mgmt__tags">
|
||||
<v-chip
|
||||
v-for="t in overview.cooccurring_tags" :key="t.id"
|
||||
size="small" @click="openTag(t.id)"
|
||||
>{{ t.name }} <span class="fc-artist-mgmt__tagc">{{ t.count }}</span></v-chip>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="overview.activity.length" class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Activity</h2>
|
||||
<svg
|
||||
class="fc-artist-mgmt__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
|
||||
preserveAspectRatio="none" role="img" aria-label="posts over time"
|
||||
>
|
||||
<polyline :points="sparkPoints" fill="none"
|
||||
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
|
||||
</svg>
|
||||
</section>
|
||||
|
||||
<section v-if="overview.sources.length" class="fc-artist-mgmt__sec">
|
||||
<div class="fc-artist-mgmt__sec-head">
|
||||
<h2 class="fc-h2">Subscriptions</h2>
|
||||
<RouterLink
|
||||
:to="`/subscriptions?artist_id=${overview.id}`"
|
||||
class="fc-artist-mgmt__manage"
|
||||
>Manage subscriptions →</RouterLink>
|
||||
</div>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="s in overview.sources" :key="s.id">
|
||||
<td>{{ s.platform }}</td>
|
||||
<td class="fc-artist-mgmt__url">{{ s.url }}</td>
|
||||
<td class="text-right">{{ s.image_count }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</section>
|
||||
|
||||
<section class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Danger zone</h2>
|
||||
<ArtistDangerZone
|
||||
:slug="overview.slug"
|
||||
:artist-id="overview.id"
|
||||
:artist-name="overview.name"
|
||||
/>
|
||||
</section>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { useRouter, RouterLink } from 'vue-router'
|
||||
|
||||
import ArtistDangerZone from './ArtistDangerZone.vue'
|
||||
|
||||
const props = defineProps({
|
||||
overview: { type: Object, required: true },
|
||||
})
|
||||
|
||||
const router = useRouter()
|
||||
|
||||
const sparkW = 600
|
||||
const sparkH = 80
|
||||
const sparkPoints = computed(() => {
|
||||
const a = props.overview.activity ?? []
|
||||
if (a.length === 0) return ''
|
||||
const max = Math.max(...a.map(p => p.count), 1)
|
||||
const stepX = a.length > 1 ? sparkW / (a.length - 1) : 0
|
||||
return a.map((p, i) => {
|
||||
const x = i * stepX
|
||||
const y = sparkH - (p.count / max) * (sparkH - 4) - 2
|
||||
return `${x.toFixed(1)},${y.toFixed(1)}`
|
||||
}).join(' ')
|
||||
})
|
||||
|
||||
function openTag (tagId) {
|
||||
router.push({ name: 'gallery', query: { tag_id: tagId } })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-artist-mgmt { padding-top: 1rem; }
|
||||
.fc-h2 {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 20px; font-weight: 500; margin-bottom: 8px;
|
||||
}
|
||||
.fc-artist-mgmt__sec { margin-bottom: 28px; }
|
||||
.fc-artist-mgmt__chips { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.fc-artist-mgmt__tags { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.fc-artist-mgmt__tagc {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin-left: 4px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.fc-artist-mgmt__spark { width: 100%; height: 80px; }
|
||||
.fc-artist-mgmt__url {
|
||||
max-width: 380px; overflow: hidden; text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.fc-artist-mgmt__sec-head {
|
||||
display: flex; align-items: baseline; justify-content: space-between;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
.fc-artist-mgmt__manage {
|
||||
font-size: 0.85rem;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
text-decoration: none;
|
||||
}
|
||||
.fc-artist-mgmt__manage:hover { text-decoration: underline; }
|
||||
</style>
|
||||
@@ -0,0 +1,90 @@
|
||||
<template>
|
||||
<div class="fc-artist-posts">
|
||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||
{{ String(store.error) }}
|
||||
</v-alert>
|
||||
|
||||
<div v-if="store.loading && store.items.length === 0" class="fc-artist-posts__loading">
|
||||
<v-progress-circular indeterminate color="accent" size="36" />
|
||||
</div>
|
||||
|
||||
<div v-else-if="store.items.length === 0 && store.done" class="fc-artist-posts__empty">
|
||||
<p>No posts for this artist yet. Switch to
|
||||
<a href="#" @click.prevent="$emit('switch-tab', 'gallery')">Gallery</a>
|
||||
to see imported images, or visit
|
||||
<RouterLink to="/subscriptions">Subscriptions</RouterLink>
|
||||
to start capturing posts.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-else>
|
||||
<PostCard v-for="p in store.items" :key="p.id" :post="p" />
|
||||
<div ref="sentinel" class="fc-artist-posts__sentinel">
|
||||
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
|
||||
<span v-else-if="store.done" class="fc-artist-posts__end">End of stream</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { RouterLink } from 'vue-router'
|
||||
|
||||
import { usePostsStore } from '../../stores/posts.js'
|
||||
import PostCard from '../posts/PostCard.vue'
|
||||
|
||||
const props = defineProps({
|
||||
artistId: { type: Number, required: true },
|
||||
})
|
||||
|
||||
defineEmits(['switch-tab'])
|
||||
|
||||
const store = usePostsStore()
|
||||
const sentinel = ref(null)
|
||||
let observer = null
|
||||
|
||||
async function reload () {
|
||||
await store.loadInitial({ artist_id: props.artistId, platform: null })
|
||||
}
|
||||
|
||||
watch(() => props.artistId, reload)
|
||||
|
||||
onMounted(async () => {
|
||||
await reload()
|
||||
observer = new IntersectionObserver((entries) => {
|
||||
if (entries.some(e => e.isIntersecting)) {
|
||||
store.loadMore()
|
||||
}
|
||||
}, { rootMargin: '400px 0px' })
|
||||
if (sentinel.value) observer.observe(sentinel.value)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
if (observer) observer.disconnect()
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-artist-posts {
|
||||
max-width: 1600px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.fc-artist-posts__loading,
|
||||
.fc-artist-posts__empty {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 2rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-artist-posts__sentinel {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 1.5rem 0;
|
||||
min-height: 2rem;
|
||||
}
|
||||
.fc-artist-posts__end {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,124 @@
|
||||
<template>
|
||||
<v-card class="fc-clean-card">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-image-size-select-small" size="small" />
|
||||
<span>Minimum dimensions</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
Find and delete images smaller than the threshold. Mirrors the
|
||||
import-time <code>min_width</code> / <code>min_height</code>
|
||||
filter, applied retroactively to the existing library.
|
||||
</p>
|
||||
|
||||
<v-row dense>
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="minW" label="Min width (px)" type="number"
|
||||
min="0" density="compact" hide-details
|
||||
/>
|
||||
</v-col>
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="minH" label="Min height (px)" type="number"
|
||||
min="0" density="compact" hide-details
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<div class="d-flex align-center mt-3" style="gap: 10px;">
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="busy"
|
||||
@click="onPreview"
|
||||
>Preview</v-btn>
|
||||
|
||||
<span v-if="preview" class="text-body-2">
|
||||
<strong>{{ preview.count }}</strong> image(s) would be deleted.
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<v-btn
|
||||
v-if="preview && preview.count > 0"
|
||||
class="mt-3"
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete"
|
||||
@click="onDeleteClick"
|
||||
>Delete {{ preview.count }} matching...</v-btn>
|
||||
</v-card-text>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-model="showModal"
|
||||
action="delete"
|
||||
kind="min-dim"
|
||||
tier="C"
|
||||
:expected-token-override="preview?.confirm_token || ''"
|
||||
:projected-counts="projectedCounts"
|
||||
:description="`Width < ${minW} OR height < ${minH}`"
|
||||
@confirm="onConfirmedDelete"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
// Backend's preview response hands the full Tier-C confirm token back
|
||||
// as `confirm_token` (e.g. `delete-min-dim-1a2b3c4d`); passed straight
|
||||
// to the modal via `expected-token-override`. We previously
|
||||
// reconstructed via Web Crypto's SHA-256, but `crypto.subtle` is
|
||||
// Secure-Context-gated and undefined on plain-HTTP origins, so the
|
||||
// Delete button silently swallowed the TypeError. Operator-flagged
|
||||
// 2026-05-27.
|
||||
|
||||
const store = useCleanupStore()
|
||||
const minW = ref(0)
|
||||
const minH = ref(0)
|
||||
const preview = ref(null)
|
||||
const busy = ref(false)
|
||||
const showModal = ref(false)
|
||||
const projectedCounts = ref({})
|
||||
|
||||
onMounted(async () => {
|
||||
await store.loadDefaults()
|
||||
minW.value = store.defaults.min_width
|
||||
minH.value = store.defaults.min_height
|
||||
})
|
||||
|
||||
async function onPreview() {
|
||||
busy.value = true
|
||||
try {
|
||||
preview.value = await store.previewMinDim(minW.value, minH.value)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Preview failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function onDeleteClick() {
|
||||
projectedCounts.value = { 'Images to delete': preview.value.count }
|
||||
showModal.value = true
|
||||
}
|
||||
|
||||
async function onConfirmedDelete(token) {
|
||||
try {
|
||||
const res = await store.deleteMinDim(minW.value, minH.value, token)
|
||||
window.__fcToast?.({
|
||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||
})
|
||||
preview.value = null
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Delete failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-clean-card { border-radius: 8px; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -0,0 +1,183 @@
|
||||
<template>
|
||||
<v-card class="fc-clean-card">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-palette-swatch" size="small" />
|
||||
<span>Single-color audit</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
Scan library for images dominated by one color within the
|
||||
tolerance. Catches placeholder / solid-fill / error-page images
|
||||
that slipped through the import filter. Same background-scan
|
||||
cadence as the transparency audit.
|
||||
</p>
|
||||
|
||||
<v-row dense>
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="threshold" label="Threshold (0–1)"
|
||||
type="number" min="0" max="1" step="0.01"
|
||||
density="compact" hide-details
|
||||
:disabled="audit && audit.status === 'running'"
|
||||
/>
|
||||
</v-col>
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="tolerance" label="Color tolerance (0–441)"
|
||||
type="number" min="0" max="441"
|
||||
density="compact" hide-details
|
||||
:disabled="audit && audit.status === 'running'"
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<v-btn
|
||||
v-if="!audit || audit.status !== 'running'"
|
||||
class="mt-3"
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify-scan"
|
||||
:loading="busy"
|
||||
@click="onStart"
|
||||
>Scan library</v-btn>
|
||||
|
||||
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||
<v-progress-linear indeterminate color="accent" />
|
||||
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||
<span>
|
||||
Scanning… {{ audit.scanned_count }} checked,
|
||||
{{ audit.matched_count }} matched
|
||||
</span>
|
||||
<v-btn
|
||||
variant="text" size="small" color="warning" rounded="pill"
|
||||
@click="onCancel"
|
||||
>Cancel</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||
<p class="text-body-2 mb-2">
|
||||
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||
image(s) match.
|
||||
</p>
|
||||
<v-btn
|
||||
v-if="audit.matched_count > 0"
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete"
|
||||
@click="onApplyClick"
|
||||
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="audit && audit.status === 'error'"
|
||||
type="error" variant="tonal" density="compact" class="mt-3"
|
||||
>Scan failed: {{ audit.error }}</v-alert>
|
||||
|
||||
<v-alert
|
||||
v-if="audit && audit.status === 'applied'"
|
||||
type="success" variant="tonal" density="compact" class="mt-3"
|
||||
>Applied — matched images deleted.</v-alert>
|
||||
</v-card-text>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-if="audit"
|
||||
v-model="showModal"
|
||||
action="delete"
|
||||
kind="audit"
|
||||
:run-id="audit.id"
|
||||
tier="C"
|
||||
:projected-counts="projectedCounts"
|
||||
description="Permanently deletes images matched by the single-color scan."
|
||||
@confirm="onConfirmedApply"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
const store = useCleanupStore()
|
||||
const threshold = ref(0.95)
|
||||
const tolerance = ref(30)
|
||||
const audit = ref(null)
|
||||
const busy = ref(false)
|
||||
const showModal = ref(false)
|
||||
const projectedCounts = ref({})
|
||||
let pollTimer = null
|
||||
|
||||
onMounted(async () => {
|
||||
await store.loadDefaults()
|
||||
threshold.value = store.defaults.single_color_threshold
|
||||
tolerance.value = store.defaults.single_color_tolerance
|
||||
})
|
||||
|
||||
onUnmounted(() => stopPoll())
|
||||
|
||||
function startPoll(id) {
|
||||
stopPoll()
|
||||
pollTimer = setInterval(async () => {
|
||||
try {
|
||||
const fresh = await store.getAudit(id)
|
||||
audit.value = fresh
|
||||
if (fresh.status !== 'running') stopPoll()
|
||||
} catch (e) {
|
||||
stopPoll()
|
||||
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
function stopPoll() {
|
||||
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||
}
|
||||
|
||||
async function onStart() {
|
||||
busy.value = true
|
||||
try {
|
||||
const res = await store.startAudit('single_color', {
|
||||
threshold: threshold.value, tolerance: tolerance.value,
|
||||
})
|
||||
audit.value = await store.getAudit(res.audit_id)
|
||||
startPoll(res.audit_id)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCancel() {
|
||||
if (!audit.value) return
|
||||
try {
|
||||
await store.cancelAudit(audit.value.id)
|
||||
audit.value = await store.getAudit(audit.value.id)
|
||||
stopPoll()
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
|
||||
function onApplyClick() {
|
||||
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||
showModal.value = true
|
||||
}
|
||||
|
||||
async function onConfirmedApply(token) {
|
||||
try {
|
||||
const res = await store.applyAudit(audit.value.id, token)
|
||||
window.__fcToast?.({
|
||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||
})
|
||||
audit.value = await store.getAudit(audit.value.id)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-clean-card { border-radius: 8px; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -0,0 +1,166 @@
|
||||
<template>
|
||||
<v-card class="fc-clean-card">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-checkerboard" size="small" />
|
||||
<span>Transparency audit</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
Scan library for images whose transparent-pixel fraction exceeds
|
||||
the threshold. Animated WebPs / GIFs are skipped (the import-side
|
||||
rule does the same). Runs as a background task — ~50ms per image,
|
||||
so a 57k library takes ~50 minutes.
|
||||
</p>
|
||||
|
||||
<v-text-field
|
||||
v-model.number="threshold" label="Transparency threshold (0–1)"
|
||||
type="number" min="0" max="1" step="0.01" density="compact" hide-details
|
||||
:disabled="audit && audit.status === 'running'"
|
||||
class="mb-3"
|
||||
/>
|
||||
|
||||
<v-btn
|
||||
v-if="!audit || audit.status !== 'running'"
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify-scan"
|
||||
:loading="busy"
|
||||
@click="onStart"
|
||||
>Scan library</v-btn>
|
||||
|
||||
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||
<v-progress-linear indeterminate color="accent" />
|
||||
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||
<span>
|
||||
Scanning… {{ audit.scanned_count }} checked,
|
||||
{{ audit.matched_count }} matched
|
||||
</span>
|
||||
<v-btn
|
||||
variant="text" size="small" color="warning" rounded="pill"
|
||||
@click="onCancel"
|
||||
>Cancel</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||
<p class="text-body-2 mb-2">
|
||||
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||
image(s) match.
|
||||
</p>
|
||||
<v-btn
|
||||
v-if="audit.matched_count > 0"
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete"
|
||||
@click="onApplyClick"
|
||||
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="audit && audit.status === 'error'"
|
||||
type="error" variant="tonal" density="compact" class="mt-3"
|
||||
>Scan failed: {{ audit.error }}</v-alert>
|
||||
|
||||
<v-alert
|
||||
v-if="audit && audit.status === 'applied'"
|
||||
type="success" variant="tonal" density="compact" class="mt-3"
|
||||
>Applied — matched images deleted.</v-alert>
|
||||
</v-card-text>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-if="audit"
|
||||
v-model="showModal"
|
||||
action="delete"
|
||||
kind="audit"
|
||||
:run-id="audit.id"
|
||||
tier="C"
|
||||
:projected-counts="projectedCounts"
|
||||
description="Permanently deletes images matched by the transparency scan."
|
||||
@confirm="onConfirmedApply"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
const store = useCleanupStore()
|
||||
const threshold = ref(0.9)
|
||||
const audit = ref(null)
|
||||
const busy = ref(false)
|
||||
const showModal = ref(false)
|
||||
const projectedCounts = ref({})
|
||||
let pollTimer = null
|
||||
|
||||
onMounted(async () => {
|
||||
await store.loadDefaults()
|
||||
threshold.value = store.defaults.transparency_threshold
|
||||
})
|
||||
|
||||
onUnmounted(() => stopPoll())
|
||||
|
||||
function startPoll(id) {
|
||||
stopPoll()
|
||||
pollTimer = setInterval(async () => {
|
||||
try {
|
||||
const fresh = await store.getAudit(id)
|
||||
audit.value = fresh
|
||||
if (fresh.status !== 'running') stopPoll()
|
||||
} catch (e) {
|
||||
stopPoll()
|
||||
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
function stopPoll() {
|
||||
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||
}
|
||||
|
||||
async function onStart() {
|
||||
busy.value = true
|
||||
try {
|
||||
const res = await store.startAudit('transparency', { threshold: threshold.value })
|
||||
audit.value = await store.getAudit(res.audit_id)
|
||||
startPoll(res.audit_id)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCancel() {
|
||||
if (!audit.value) return
|
||||
try {
|
||||
await store.cancelAudit(audit.value.id)
|
||||
audit.value = await store.getAudit(audit.value.id)
|
||||
stopPoll()
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
|
||||
function onApplyClick() {
|
||||
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||
showModal.value = true
|
||||
}
|
||||
|
||||
async function onConfirmedApply(token) {
|
||||
try {
|
||||
const res = await store.applyAudit(audit.value.id, token)
|
||||
window.__fcToast?.({
|
||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||
})
|
||||
audit.value = await store.getAudit(audit.value.id)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-clean-card { border-radius: 8px; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -0,0 +1,172 @@
|
||||
<template>
|
||||
<v-dialog :model-value="modelValue" max-width="900"
|
||||
@update:model-value="$emit('update:modelValue', $event)">
|
||||
<v-card>
|
||||
<v-card-title class="d-flex align-center" style="gap: 12px;">
|
||||
<v-icon icon="mdi-alert-circle-outline" color="error" />
|
||||
<span>{{ displayTitle }}</span>
|
||||
<v-spacer />
|
||||
<v-btn icon variant="text" size="small" @click="close">
|
||||
<v-icon>mdi-close</v-icon>
|
||||
</v-btn>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<dl v-if="contextRows.length" class="fc-err-context">
|
||||
<template v-for="(row, idx) in contextRows" :key="idx">
|
||||
<dt>{{ row[0] }}</dt>
|
||||
<dd>{{ row[1] }}</dd>
|
||||
</template>
|
||||
</dl>
|
||||
<pre class="fc-err-pre">{{ displayMessage || '(no error message)' }}</pre>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small"
|
||||
:prepend-icon="copied ? 'mdi-check' : 'mdi-content-copy'"
|
||||
@click="onCopy"
|
||||
>{{ copied ? 'Copied' : 'Copy' }}</v-btn>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" rounded="pill" @click="close">Close</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
import { copyText } from '../../utils/clipboard.js'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, default: false },
|
||||
// Legacy mode: pass title + message strings. Used by callers whose row
|
||||
// shape lacks structured context (e.g. ImportTaskList where `error` is
|
||||
// a plain string on the import_task row, not a TaskRun).
|
||||
title: { type: String, default: 'Error details' },
|
||||
message: { type: String, default: '' },
|
||||
// Row mode: pass the full TaskRun-shaped dict from /api/system_activity.
|
||||
// When set, displayTitle/displayMessage derive from the row and a context
|
||||
// panel of task_name/queue/target/duration/etc. renders above the error.
|
||||
row: { type: Object, default: null },
|
||||
})
|
||||
|
||||
const emit = defineEmits(['update:modelValue'])
|
||||
|
||||
const copied = ref(false)
|
||||
let copiedTimer = null
|
||||
|
||||
const displayTitle = computed(() => {
|
||||
if (props.row) return props.row.error_type || 'Error details'
|
||||
return props.title
|
||||
})
|
||||
|
||||
const displayMessage = computed(() => {
|
||||
if (props.row) return props.row.error_message || ''
|
||||
return props.message
|
||||
})
|
||||
|
||||
function _shortTaskName (name) {
|
||||
if (!name) return ''
|
||||
const parts = String(name).split('.')
|
||||
return parts[parts.length - 1]
|
||||
}
|
||||
|
||||
function _formatDuration (ms) {
|
||||
if (ms == null) return null
|
||||
if (ms < 1000) return `${ms} ms`
|
||||
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
|
||||
return `${(ms / 60_000).toFixed(1)} min`
|
||||
}
|
||||
|
||||
const contextRows = computed(() => {
|
||||
const r = props.row
|
||||
if (!r) return []
|
||||
const rows = []
|
||||
if (r.task_name) rows.push(['Task', _shortTaskName(r.task_name)])
|
||||
if (r.queue) rows.push(['Queue', r.queue])
|
||||
if (r.target_id != null) rows.push(['Target', r.target_id])
|
||||
const dur = _formatDuration(r.duration_ms)
|
||||
if (dur != null) rows.push(['Duration', dur])
|
||||
if (r.started_at) rows.push(['Started', r.started_at])
|
||||
if (r.finished_at) rows.push(['Finished', r.finished_at])
|
||||
if (r.retry_count) rows.push(['Retries', r.retry_count])
|
||||
if (r.worker_hostname) rows.push(['Worker', r.worker_hostname])
|
||||
if (r.celery_task_id) rows.push(['Celery ID', r.celery_task_id])
|
||||
if (r.args_summary) rows.push(['Args', r.args_summary])
|
||||
return rows
|
||||
})
|
||||
|
||||
watch(() => props.modelValue, (open) => {
|
||||
if (!open) {
|
||||
copied.value = false
|
||||
if (copiedTimer) { clearTimeout(copiedTimer); copiedTimer = null }
|
||||
}
|
||||
})
|
||||
|
||||
function close () {
|
||||
emit('update:modelValue', false)
|
||||
}
|
||||
|
||||
async function onCopy () {
|
||||
let text = displayMessage.value || ''
|
||||
if (contextRows.value.length) {
|
||||
const header = contextRows.value.map(([k, v]) => `${k}: ${v}`).join('\n')
|
||||
text = `${header}\n\nError: ${displayTitle.value}\n${text}`
|
||||
}
|
||||
try {
|
||||
await copyText(text)
|
||||
copied.value = true
|
||||
if (copiedTimer) clearTimeout(copiedTimer)
|
||||
copiedTimer = setTimeout(() => { copied.value = false }, 1500)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* Context panel: muted labels in vellum, crisp values in parchment. The
|
||||
2-column key/value grid keeps rows visually scannable when there are
|
||||
many fields (Task, Queue, Target, Duration, Started, Finished, Retries,
|
||||
Worker, Celery ID, Args). */
|
||||
.fc-err-context {
|
||||
display: grid;
|
||||
grid-template-columns: max-content 1fr;
|
||||
column-gap: 14px;
|
||||
row-gap: 4px;
|
||||
margin: 0 0 14px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
.fc-err-context dt {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-weight: 500;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.fc-err-context dd {
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin: 0;
|
||||
font-variant-numeric: tabular-nums;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
/* Error pre block: high-contrast pairing. The page's `background` token
|
||||
(obsidian #14171A) is darker than the modal card's `surface` (iron
|
||||
#1E2228), so parchment text reads crisply against it. The prior pairing
|
||||
used `surface-variant` which Vuetify auto-derives to a near-parchment
|
||||
light value in this theme — pale-on-pale and unreadable. Operator-
|
||||
flagged 2026-05-26 ("ui contrast is poor"). */
|
||||
.fc-err-pre {
|
||||
font-family: 'JetBrains Mono', ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
background: rgb(var(--v-theme-background));
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
padding: 12px 14px;
|
||||
border-radius: 6px;
|
||||
max-height: 60vh;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
}
|
||||
</style>
|
||||
@@ -28,6 +28,7 @@
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||
import { copyText } from '../../utils/clipboard.js'
|
||||
|
||||
const store = useCredentialsStore()
|
||||
const showRotateConfirm = ref(false)
|
||||
@@ -37,7 +38,7 @@ onMounted(() => store.loadKey())
|
||||
async function copyKey() {
|
||||
if (!store.extensionKey) return
|
||||
try {
|
||||
await navigator.clipboard.writeText(store.extensionKey)
|
||||
await copyText(store.extensionKey)
|
||||
globalThis.window?.__fcToast?.({ text: 'Copied', type: 'success' })
|
||||
} catch {
|
||||
globalThis.window?.__fcToast?.({ text: 'Copy failed', type: 'error' })
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
<div v-for="(col, ci) in columns" :key="ci" class="fc-masonry__col">
|
||||
<button
|
||||
v-for="item in col" :key="item.id"
|
||||
class="fc-masonry__item" type="button"
|
||||
class="fc-masonry__item"
|
||||
:class="{ 'fc-masonry__item--anim': shouldAnimate(item) }"
|
||||
:style="itemStyle(item)"
|
||||
type="button"
|
||||
@click="$emit('open', item.id)"
|
||||
>
|
||||
<img
|
||||
@@ -33,7 +36,13 @@ import { usePolyMasonry } from '../../composables/usePolyMasonry.js'
|
||||
const props = defineProps({
|
||||
items: { type: Array, default: () => [] },
|
||||
loading: { type: Boolean, default: false },
|
||||
hasMore: { type: Boolean, default: false }
|
||||
hasMore: { type: Boolean, default: false },
|
||||
// Items at indices >= animateFromIndex get the stagger fade-in. Opt-in
|
||||
// — defaults to Infinity (no animation) so views that don't want it
|
||||
// (ArtistView, etc.) don't pay the layout-shift cost. ShowcaseView
|
||||
// uses 0 on initial load / shuffle and prevCount on infinite-scroll
|
||||
// appends.
|
||||
animateFromIndex: { type: Number, default: Number.POSITIVE_INFINITY },
|
||||
})
|
||||
const emit = defineEmits(['load-more', 'open'])
|
||||
|
||||
@@ -43,6 +52,25 @@ const { columnCount, distribute } = usePolyMasonry(containerEl)
|
||||
|
||||
const columns = computed(() => distribute(props.items, columnCount.value))
|
||||
|
||||
// id → index lookup so we can derive the stagger from natural reading
|
||||
// order even after the masonry distributes items across columns.
|
||||
const idxById = computed(() => {
|
||||
const m = new Map()
|
||||
props.items.forEach((it, i) => m.set(it.id, i))
|
||||
return m
|
||||
})
|
||||
|
||||
function shouldAnimate(item) {
|
||||
const idx = idxById.value.get(item.id)
|
||||
return idx !== undefined && idx >= props.animateFromIndex
|
||||
}
|
||||
|
||||
function itemStyle(item) {
|
||||
if (!shouldAnimate(item)) return {}
|
||||
const idx = idxById.value.get(item.id) - props.animateFromIndex
|
||||
return { '--stagger-index': idx }
|
||||
}
|
||||
|
||||
function aspectStyle(item) {
|
||||
const w = Number(item.width)
|
||||
const h = Number(item.height)
|
||||
@@ -81,4 +109,23 @@ onUnmounted(() => observer && observer.disconnect())
|
||||
display: flex; justify-content: center; padding: 32px 0; min-height: 60px;
|
||||
}
|
||||
.fc-masonry__end { text-align: center; padding: 32px 0; }
|
||||
|
||||
/* IR-parity stagger fade-in for showcase entry / shuffle. 60ms between
|
||||
items, 250ms each — matches IR's `itemFadeIn` keyframe (style.css
|
||||
~line 1834). Honors prefers-reduced-motion. */
|
||||
@keyframes fc-masonry-item-in {
|
||||
from { opacity: 0; transform: translateY(12px); }
|
||||
to { opacity: 1; transform: translateY(0); }
|
||||
}
|
||||
.fc-masonry__item--anim {
|
||||
animation: fc-masonry-item-in 0.25s ease forwards;
|
||||
animation-delay: calc(var(--stagger-index, 0) * 60ms);
|
||||
opacity: 0;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.fc-masonry__item--anim {
|
||||
animation: none;
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,47 +1,118 @@
|
||||
<template>
|
||||
<div class="fc-dl-row" @click="$emit('open', event.id)">
|
||||
<v-icon :icon="statusIcon" :color="statusColor" size="small" />
|
||||
<div
|
||||
class="fc-dl-row"
|
||||
:class="[`fc-dl-row--${event.status || 'unknown'}`]"
|
||||
@click="$emit('open', event.id)"
|
||||
>
|
||||
<!-- Colored left edge marks the run's status; matches the row's
|
||||
status-chip color but reads at a glance without needing to
|
||||
parse the chip text. -->
|
||||
<div class="fc-dl-row__bar" />
|
||||
|
||||
<v-chip
|
||||
:color="statusColor"
|
||||
size="small"
|
||||
variant="tonal"
|
||||
:prepend-icon="statusIcon"
|
||||
class="fc-dl-row__status"
|
||||
>{{ statusLabel }}</v-chip>
|
||||
|
||||
<RouterLink
|
||||
v-if="event.artist_slug"
|
||||
:to="`/artist/${event.artist_slug}`"
|
||||
class="fc-dl-row__artist"
|
||||
@click.stop
|
||||
>{{ event.artist_name }}</RouterLink>
|
||||
<span v-else class="fc-dl-row__artist">—</span>
|
||||
<v-chip size="x-small" variant="tonal">{{ event.platform || '—' }}</v-chip>
|
||||
<span class="fc-dl-row__time">{{ fmtTime(event.started_at) }}</span>
|
||||
<span class="fc-dl-row__files">{{ event.files_count }} files</span>
|
||||
<span class="fc-dl-row__duration">{{ fmtDuration(event.summary?.duration_seconds) }}</span>
|
||||
<span v-if="event.error" class="fc-dl-row__error">{{ event.error }}</span>
|
||||
<span v-else class="fc-dl-row__artist fc-dl-row__artist--missing">—</span>
|
||||
|
||||
<PlatformChip
|
||||
v-if="event.platform"
|
||||
:platform="event.platform"
|
||||
size="x-small"
|
||||
class="fc-dl-row__platform"
|
||||
/>
|
||||
<span v-else class="fc-dl-row__platform-missing">—</span>
|
||||
|
||||
<span class="fc-dl-row__time" :title="event.started_at">
|
||||
{{ fmtTime(event.started_at) }}
|
||||
</span>
|
||||
|
||||
<v-chip
|
||||
v-if="event.files_count > 0"
|
||||
size="x-small" variant="tonal" color="info"
|
||||
prepend-icon="mdi-image-multiple"
|
||||
class="fc-dl-row__files"
|
||||
>{{ event.files_count }}</v-chip>
|
||||
<span v-else class="fc-dl-row__no-files" aria-label="no new files">·</span>
|
||||
|
||||
<span class="fc-dl-row__duration">
|
||||
{{ fmtDuration(event.summary?.duration_seconds) }}
|
||||
</span>
|
||||
|
||||
<v-chip
|
||||
v-if="event.error"
|
||||
color="error" size="x-small" variant="tonal"
|
||||
prepend-icon="mdi-alert-octagon"
|
||||
class="fc-dl-row__error"
|
||||
:title="event.error"
|
||||
>{{ truncateError(event.error) }}</v-chip>
|
||||
<span v-else class="fc-dl-row__error-spacer" />
|
||||
|
||||
<div class="fc-dl-row__actions" @click.stop>
|
||||
<v-btn
|
||||
v-if="event.status === 'error' && event.source_id"
|
||||
icon size="x-small" variant="text" color="warning"
|
||||
:loading="retrying"
|
||||
@click.stop="onRetry"
|
||||
>
|
||||
<v-icon size="small">mdi-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Retry source check</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn icon size="x-small" variant="text" @click.stop="$emit('open', event.id)">
|
||||
<v-icon size="small">mdi-information-outline</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Details</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
v-if="event.artist_slug"
|
||||
icon size="x-small" variant="text"
|
||||
:to="`/artist/${event.artist_slug}`"
|
||||
@click.stop
|
||||
>
|
||||
<v-icon size="small">mdi-account-circle</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Open artist</v-tooltip>
|
||||
</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import { RouterLink } from 'vue-router'
|
||||
|
||||
import PlatformChip from '../subscriptions/PlatformChip.vue'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
|
||||
const props = defineProps({ event: { type: Object, required: true } })
|
||||
defineEmits(['open'])
|
||||
|
||||
const statusIcon = computed(() => ({
|
||||
ok: 'mdi-check-circle',
|
||||
error: 'mdi-alert-circle',
|
||||
running: 'mdi-progress-clock',
|
||||
pending: 'mdi-clock-outline',
|
||||
skipped: 'mdi-minus-circle',
|
||||
}[props.event.status] || 'mdi-help-circle'))
|
||||
const sourcesStore = useSourcesStore()
|
||||
const retrying = ref(false)
|
||||
|
||||
const statusColor = computed(() => ({
|
||||
ok: 'success',
|
||||
error: 'error',
|
||||
running: 'info',
|
||||
pending: 'secondary',
|
||||
skipped: 'warning',
|
||||
}[props.event.status] || undefined))
|
||||
const _STATUS = {
|
||||
ok: { color: 'success', icon: 'mdi-check-circle', label: 'Completed' },
|
||||
error: { color: 'error', icon: 'mdi-alert-circle', label: 'Failed' },
|
||||
running: { color: 'info', icon: 'mdi-progress-clock', label: 'Running' },
|
||||
pending: { color: 'grey', icon: 'mdi-clock-outline', label: 'Queued' },
|
||||
skipped: { color: 'warning', icon: 'mdi-skip-next', label: 'Skipped' },
|
||||
}
|
||||
const statusColor = computed(() => _STATUS[props.event.status]?.color || 'grey')
|
||||
const statusIcon = computed(() => _STATUS[props.event.status]?.icon || 'mdi-help-circle')
|
||||
const statusLabel = computed(() => _STATUS[props.event.status]?.label || props.event.status)
|
||||
|
||||
function fmtTime(iso) {
|
||||
if (!iso) return '—'
|
||||
return iso.slice(0, 19).replace('T', ' ')
|
||||
// 2026-05-27 23:36 — second granularity is in the row's title attr
|
||||
return iso.slice(0, 16).replace('T', ' ')
|
||||
}
|
||||
function fmtDuration(sec) {
|
||||
if (sec == null) return '—'
|
||||
@@ -49,34 +120,107 @@ function fmtDuration(sec) {
|
||||
const m = Math.floor(sec / 60), s = Math.floor(sec % 60)
|
||||
return `${m}m ${s}s`
|
||||
}
|
||||
function truncateError(msg) {
|
||||
const s = String(msg || '')
|
||||
if (s.length <= 60) return s
|
||||
return s.slice(0, 57) + '…'
|
||||
}
|
||||
|
||||
async function onRetry() {
|
||||
if (!props.event.source_id) return
|
||||
retrying.value = true
|
||||
try {
|
||||
await sourcesStore.checkNow(props.event.source_id)
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: `Source check re-queued`, type: 'success',
|
||||
})
|
||||
} catch (e) {
|
||||
const isInFlight = !!e?.body?.download_event_id
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: isInFlight ? 'Already running' : `Retry failed: ${e?.detail || e?.message || e}`,
|
||||
type: isInFlight ? 'info' : 'error',
|
||||
})
|
||||
} finally {
|
||||
retrying.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dl-row {
|
||||
position: relative;
|
||||
display: grid;
|
||||
grid-template-columns: 24px 1fr 96px 160px 80px 80px 1fr;
|
||||
gap: 0.75rem;
|
||||
grid-template-columns:
|
||||
/* bar */ 4px
|
||||
/* status */ 120px
|
||||
/* artist */ minmax(120px, 1.2fr)
|
||||
/* plat */ 140px
|
||||
/* time */ 140px
|
||||
/* files */ 60px
|
||||
/* dur */ 70px
|
||||
/* error */ minmax(0, 1.5fr)
|
||||
/* actions*/ 120px;
|
||||
gap: 0.6rem;
|
||||
align-items: center;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.18);
|
||||
padding: 0.55rem 0.75rem 0.55rem 0;
|
||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||
cursor: pointer;
|
||||
transition: background 0.12s ease;
|
||||
}
|
||||
.fc-dl-row:hover { background: rgb(var(--v-theme-surface) / 0.5); }
|
||||
.fc-dl-row:hover {
|
||||
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||
}
|
||||
.fc-dl-row__bar {
|
||||
width: 4px;
|
||||
align-self: stretch;
|
||||
border-radius: 0 2px 2px 0;
|
||||
}
|
||||
.fc-dl-row--ok .fc-dl-row__bar { background: rgb(var(--v-theme-success)); }
|
||||
.fc-dl-row--error .fc-dl-row__bar { background: rgb(var(--v-theme-error)); }
|
||||
.fc-dl-row--running .fc-dl-row__bar { background: rgb(var(--v-theme-info)); }
|
||||
.fc-dl-row--skipped .fc-dl-row__bar { background: rgb(var(--v-theme-warning)); }
|
||||
.fc-dl-row--pending .fc-dl-row__bar { background: rgb(var(--v-theme-on-surface-variant) / 0.4); }
|
||||
|
||||
.fc-dl-row__status { justify-self: start; }
|
||||
.fc-dl-row__artist {
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
text-decoration: none;
|
||||
font-weight: 500;
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-dl-row__artist--missing,
|
||||
.fc-dl-row__platform-missing,
|
||||
.fc-dl-row__no-files {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
opacity: 0.5;
|
||||
}
|
||||
.fc-dl-row__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||
.fc-dl-row__time, .fc-dl-row__files, .fc-dl-row__duration {
|
||||
.fc-dl-row__platform { justify-self: start; }
|
||||
.fc-dl-row__time,
|
||||
.fc-dl-row__duration {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-size: 0.85rem;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.fc-dl-row__error {
|
||||
color: rgb(var(--v-theme-error));
|
||||
font-size: 0.85rem;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.fc-dl-row__no-files {
|
||||
text-align: center;
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
.fc-dl-row__error {
|
||||
justify-self: start;
|
||||
max-width: 100%;
|
||||
}
|
||||
.fc-dl-row__error :deep(.v-chip__content) {
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-dl-row__error-spacer { /* keeps the grid column reserved */ }
|
||||
|
||||
.fc-dl-row__actions {
|
||||
display: flex; gap: 2px;
|
||||
justify-self: end;
|
||||
opacity: 0.5;
|
||||
transition: opacity 0.12s ease;
|
||||
}
|
||||
.fc-dl-row:hover .fc-dl-row__actions { opacity: 1; }
|
||||
</style>
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
v-model="deleteModalOpen"
|
||||
action="delete"
|
||||
kind="images-selection"
|
||||
:run-id="bulkToken"
|
||||
:expected-token-override="bulkProjected?.confirm_token || ''"
|
||||
tier="C"
|
||||
:projected-counts="bulkProjectedCounts"
|
||||
:description="bulkDescription"
|
||||
@@ -130,7 +130,6 @@ watch(() => sel.order.length, () => {
|
||||
const deleting = ref(false)
|
||||
const deleteModalOpen = ref(false)
|
||||
const bulkProjected = ref(null)
|
||||
const bulkToken = ref('')
|
||||
|
||||
const bulkProjectedCounts = computed(() => bulkProjected.value
|
||||
? {
|
||||
@@ -147,24 +146,22 @@ const bulkDescription = computed(
|
||||
: '',
|
||||
)
|
||||
|
||||
async function _computeSha8(ids) {
|
||||
const canon = [...ids].sort((a, b) => a - b).join(',')
|
||||
const buf = new TextEncoder().encode(canon)
|
||||
const hashBuf = await crypto.subtle.digest('SHA-256', buf)
|
||||
const bytes = new Uint8Array(hashBuf)
|
||||
let hex = ''
|
||||
for (let i = 0; i < 4; i++) {
|
||||
hex += bytes[i].toString(16).padStart(2, '0')
|
||||
}
|
||||
return hex
|
||||
}
|
||||
// The dry-run response hands the canonical Tier-C confirm token back
|
||||
// as `confirm_token` (e.g. `delete-images-1a2b3c4d`), passed straight
|
||||
// to the modal via `expected-token-override`. We used to compute the
|
||||
// hash client-side via `crypto.subtle.digest`, but (1) that's
|
||||
// Secure-Context-gated and undefined on plain-HTTP origins
|
||||
// (homelab posture), so the click silently threw TypeError and the
|
||||
// modal never opened, and (2) the modal's `kind="images-selection"`
|
||||
// produced `delete-images-selection-<sha8>` while the backend
|
||||
// expected `delete-images-<sha8>` — so it never would have worked
|
||||
// even on HTTPS. Operator-flagged 2026-05-27.
|
||||
|
||||
async function onDeleteClick() {
|
||||
if (!sel.order.length) return
|
||||
deleting.value = true
|
||||
try {
|
||||
bulkProjected.value = await adminStore.projectBulkImageDelete(sel.order)
|
||||
bulkToken.value = await _computeSha8(sel.order)
|
||||
deleteModalOpen.value = true
|
||||
} finally {
|
||||
deleting.value = false
|
||||
|
||||
@@ -58,17 +58,27 @@ import { computed, ref, watch } from 'vue'
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, required: true },
|
||||
action: { type: String, required: true }, // 'restore' | 'delete'
|
||||
kind: { type: String, required: true }, // 'db' | 'images' | 'artist' | 'tag' | 'images-selection'
|
||||
kind: { type: String, required: true }, // 'db' | 'images' | 'artist' | 'tag' | 'images-selection' | 'audit' | 'min-dim'
|
||||
runId: { type: [Number, String], default: '' }, // numeric id or sha8 string
|
||||
description: { type: String, default: '' },
|
||||
tier: { type: String, default: 'C' }, // 'B' | 'C'
|
||||
projectedCounts: { type: Object, default: null },
|
||||
// Override the `${action}-${kind}-${runId}` token formula. Use when
|
||||
// the backend computes the canonical confirm token (e.g. bulk-delete
|
||||
// and min-dim cleanup both return `confirm_token` from their dry-run
|
||||
// endpoints) and the UI's kind/runId would otherwise produce a
|
||||
// mismatched string. Operator-flagged 2026-05-27 after the
|
||||
// BulkEditor's kind="images-selection" produced
|
||||
// `delete-images-selection-<sha8>` while the backend expected
|
||||
// `delete-images-<sha8>`.
|
||||
expectedTokenOverride: { type: String, default: '' },
|
||||
})
|
||||
const emit = defineEmits(['update:modelValue', 'confirm'])
|
||||
|
||||
const typed = ref('')
|
||||
const expectedToken = computed(
|
||||
() => `${props.action}-${props.kind}-${props.runId}`,
|
||||
() => props.expectedTokenOverride
|
||||
|| `${props.action}-${props.kind}-${props.runId}`,
|
||||
)
|
||||
const titleVerb = computed(
|
||||
() => props.action === 'restore' ? 'Restore' : 'Delete',
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
<template>
|
||||
<div class="fc-tag-autocomplete">
|
||||
<div class="d-flex" style="gap: 6px;">
|
||||
<v-select
|
||||
v-model="kind" :items="kindOptions" :item-title="(k) => k.label" :item-value="(k) => k.value"
|
||||
density="compact" hide-details style="max-width: 140px;"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="query" placeholder="Add tag…" density="compact" hide-details
|
||||
@keydown.down.prevent="moveHighlight(1)" @keydown.up.prevent="moveHighlight(-1)"
|
||||
@keydown.enter.prevent="onEnter" @keydown.esc="$emit('cancel')"
|
||||
/>
|
||||
</div>
|
||||
<v-list v-if="hits.length || allowCreate" density="compact" class="fc-tag-autocomplete__list">
|
||||
<v-text-field
|
||||
v-model="query"
|
||||
placeholder="Add tag (or kind:name — character/fandom/series)"
|
||||
density="compact" hide-details
|
||||
@keydown.down.prevent="moveHighlight(1)"
|
||||
@keydown.up.prevent="moveHighlight(-1)"
|
||||
@keydown.enter.prevent="onEnter"
|
||||
@keydown.esc="$emit('cancel')"
|
||||
/>
|
||||
<v-list
|
||||
v-if="hits.length || allowCreate"
|
||||
density="compact" class="fc-tag-autocomplete__list"
|
||||
>
|
||||
<v-list-item
|
||||
v-for="(h, idx) in hits" :key="h.id"
|
||||
:active="idx === highlight" @click="onPick(h)"
|
||||
@@ -26,7 +27,7 @@
|
||||
<span v-if="h.fandom_name" class="text-caption">— {{ h.fandom_name }}</span>
|
||||
</v-list-item-title>
|
||||
<template #append>
|
||||
<span class="text-caption">{{ h.image_count }}</span>
|
||||
<span class="text-caption">{{ h.kind }}</span>
|
||||
</template>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
@@ -34,9 +35,13 @@
|
||||
@click="onCreate"
|
||||
>
|
||||
<template #prepend>
|
||||
<v-icon size="small" :color="store.colorFor(kind)">{{ iconFor(kind) }}</v-icon>
|
||||
<v-icon size="small" :color="store.colorFor(parsedKind)">
|
||||
{{ iconFor(parsedKind) }}
|
||||
</v-icon>
|
||||
</template>
|
||||
<v-list-item-title>Create "{{ query }}" as {{ kind }}</v-list-item-title>
|
||||
<v-list-item-title>
|
||||
Create "{{ parsedName }}" as {{ parsedKind }}
|
||||
</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
|
||||
@@ -54,65 +59,95 @@ import FandomPicker from './FandomPicker.vue'
|
||||
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
||||
const store = useTagStore()
|
||||
|
||||
const kind = ref('general')
|
||||
// Single text input; no kind dropdown. Client-side mirror of the
|
||||
// backend's parse_kind_prefix lives below — kept in sync with
|
||||
// KNOWN_KINDS in backend/app/utils/tag_prefix.py. The backend is the
|
||||
// canonical parser; this mirror just powers the live Create-label
|
||||
// preview ("Create 'Eric' as artist") and the autocomplete query.
|
||||
const query = ref('')
|
||||
const hits = ref([])
|
||||
const highlight = ref(0)
|
||||
const fandomDialog = ref(false)
|
||||
let pendingNewName = null
|
||||
|
||||
const kindOptions = store.kindOptions()
|
||||
const KNOWN_KINDS = new Set([
|
||||
'character', 'fandom', 'series',
|
||||
])
|
||||
const KIND_ICONS = {
|
||||
general: 'mdi-tag', artist: 'mdi-palette', character: 'mdi-account-circle',
|
||||
general: 'mdi-tag', character: 'mdi-account-circle',
|
||||
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
|
||||
meta: 'mdi-cog-outline', rating: 'mdi-shield-check-outline'
|
||||
}
|
||||
function iconFor(k) { return KIND_ICONS[k] || 'mdi-tag' }
|
||||
function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
||||
|
||||
const parsed = computed(() => {
|
||||
const raw = query.value.trim()
|
||||
if (raw.includes(':')) {
|
||||
const idx = raw.indexOf(':')
|
||||
const prefix = raw.slice(0, idx).toLowerCase()
|
||||
if (KNOWN_KINDS.has(prefix)) {
|
||||
return { kind: prefix, name: raw.slice(idx + 1).trim() }
|
||||
}
|
||||
}
|
||||
return { kind: 'general', name: raw }
|
||||
})
|
||||
const parsedKind = computed(() => parsed.value.kind)
|
||||
const parsedName = computed(() => parsed.value.name)
|
||||
|
||||
let debounceId = null
|
||||
watch([query, kind], () => {
|
||||
watch(query, () => {
|
||||
highlight.value = 0
|
||||
if (debounceId) clearTimeout(debounceId)
|
||||
debounceId = setTimeout(async () => {
|
||||
const q = query.value.trim()
|
||||
const q = parsedName.value
|
||||
if (!q) { hits.value = []; return }
|
||||
hits.value = await store.autocomplete(q, kind.value, 10)
|
||||
// Autocomplete across ALL kinds. When the user typed a prefix the
|
||||
// matches list is naturally narrower because the parsed name is
|
||||
// shorter; we don't filter server-side by kind.
|
||||
hits.value = await store.autocomplete(q, null, 10)
|
||||
}, 200)
|
||||
})
|
||||
|
||||
const allowCreate = computed(() => {
|
||||
const q = query.value.trim()
|
||||
return q && !hits.value.some(h => h.name.toLowerCase() === q.toLowerCase() && h.kind === kind.value)
|
||||
const q = parsedName.value
|
||||
if (!q) return false
|
||||
return !hits.value.some(h =>
|
||||
h.name.toLowerCase() === q.toLowerCase() && h.kind === parsedKind.value,
|
||||
)
|
||||
})
|
||||
|
||||
function moveHighlight(delta) {
|
||||
function moveHighlight (delta) {
|
||||
const total = hits.value.length + (allowCreate.value ? 1 : 0)
|
||||
if (total === 0) return
|
||||
highlight.value = (highlight.value + delta + total) % total
|
||||
}
|
||||
|
||||
function onPick(hit) { emit('pick-existing', hit); reset() }
|
||||
function onPick (hit) { emit('pick-existing', hit); reset() }
|
||||
|
||||
function onCreate() {
|
||||
const name = query.value.trim()
|
||||
if (kind.value === 'character') {
|
||||
// Character requires a fandom — open the picker.
|
||||
function onCreate () {
|
||||
const name = parsedName.value
|
||||
const kind = parsedKind.value
|
||||
if (kind === 'character') {
|
||||
pendingNewName = name
|
||||
fandomDialog.value = true
|
||||
return
|
||||
}
|
||||
emit('pick-new', { name, kind: kind.value, fandom_id: null })
|
||||
// Pass explicit kind here; the backend accepts both shapes. Passing
|
||||
// it makes the parsed kind preview match the actual server outcome
|
||||
// for users who didn't use a prefix (general goes through cleanly).
|
||||
emit('pick-new', { name, kind, fandom_id: null })
|
||||
reset()
|
||||
}
|
||||
|
||||
function onFandomChosen(fandom) {
|
||||
function onFandomChosen (fandom) {
|
||||
fandomDialog.value = false
|
||||
emit('pick-new', { name: pendingNewName, kind: 'character', fandom_id: fandom.id })
|
||||
emit('pick-new', {
|
||||
name: pendingNewName, kind: 'character', fandom_id: fandom.id,
|
||||
})
|
||||
pendingNewName = null
|
||||
reset()
|
||||
}
|
||||
|
||||
function onEnter() {
|
||||
function onEnter () {
|
||||
if (highlight.value < hits.value.length) {
|
||||
onPick(hits.value[highlight.value])
|
||||
} else if (allowCreate.value) {
|
||||
@@ -120,7 +155,7 @@ function onEnter() {
|
||||
}
|
||||
}
|
||||
|
||||
function reset() { query.value = ''; hits.value = []; highlight.value = 0 }
|
||||
function reset () { query.value = ''; hits.value = []; highlight.value = 0 }
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -1,75 +1,133 @@
|
||||
<template>
|
||||
<v-card class="fc-post-card" variant="outlined">
|
||||
<v-card
|
||||
:class="['fc-post-card', expanded && 'fc-post-card--expanded']"
|
||||
variant="outlined"
|
||||
:tabindex="expanded ? -1 : 0"
|
||||
@click="onCardClick"
|
||||
@keydown.enter="onCardClick"
|
||||
>
|
||||
<div class="fc-post-card__head">
|
||||
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
||||
<RouterLink
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
class="fc-post-card__artist"
|
||||
@click.stop
|
||||
>{{ post.artist.name }}</RouterLink>
|
||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||
<span v-if="expanded && images.length" class="fc-post-card__meta">
|
||||
· {{ images.length }} image{{ images.length === 1 ? '' : 's' }}
|
||||
</span>
|
||||
<span v-if="expanded && attachments.length" class="fc-post-card__meta">
|
||||
· {{ attachments.length }} attachment{{ attachments.length === 1 ? '' : 's' }}
|
||||
</span>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
v-if="post.post_url"
|
||||
:href="post.post_url" target="_blank" rel="noopener"
|
||||
icon="mdi-open-in-new" size="x-small" variant="text"
|
||||
:aria-label="`open original post on ${post.source.platform}`"
|
||||
@click.stop
|
||||
/>
|
||||
<v-btn
|
||||
:icon="expanded ? 'mdi-chevron-up' : 'mdi-chevron-down'"
|
||||
size="x-small" variant="text"
|
||||
:aria-label="expanded ? 'Collapse post' : 'Expand post'"
|
||||
@click.stop="toggleExpanded"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<h3 v-if="post.post_title" class="fc-post-card__title">{{ post.post_title }}</h3>
|
||||
<!-- Compact body: collapsed card. Hero + thumb rail + truncated text. -->
|
||||
<div v-if="!expanded" class="fc-post-card__body">
|
||||
<div class="fc-post-card__media">
|
||||
<template v-if="images.length">
|
||||
<div class="fc-post-card__hero">
|
||||
<img :src="hero.thumbnail_url" :alt="`hero thumbnail`" loading="lazy" />
|
||||
</div>
|
||||
<div v-if="rail.length" class="fc-post-card__rail">
|
||||
<div v-for="t in rail" :key="t.image_id" class="fc-post-card__rail-cell">
|
||||
<img :src="t.thumbnail_url" :alt="`thumbnail`" loading="lazy" />
|
||||
</div>
|
||||
<div v-if="moreCount > 0" class="fc-post-card__rail-more">
|
||||
+{{ moreCount }}
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<PostEmptyThumbs v-else />
|
||||
</div>
|
||||
|
||||
<div v-if="descriptionToShow" class="fc-post-card__desc">
|
||||
<span class="fc-post-card__desc-text">{{ descriptionToShow }}</span>
|
||||
<button
|
||||
v-if="post.description_truncated && !expanded"
|
||||
class="fc-post-card__more" type="button"
|
||||
@click="expand"
|
||||
>Show more</button>
|
||||
<button
|
||||
v-if="expanded"
|
||||
class="fc-post-card__more" type="button"
|
||||
@click="expanded = false"
|
||||
>Show less</button>
|
||||
<div class="fc-post-card__text">
|
||||
<h3 v-if="post.post_title" class="fc-post-card__title">
|
||||
{{ post.post_title }}
|
||||
</h3>
|
||||
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
||||
Post {{ post.external_post_id }}
|
||||
</h3>
|
||||
|
||||
<p v-if="post.description_plain" class="fc-post-card__desc">
|
||||
{{ post.description_plain }}
|
||||
</p>
|
||||
<p v-else class="fc-post-card__desc fc-post-card__desc--missing">
|
||||
(no description)
|
||||
</p>
|
||||
|
||||
<div v-if="post.attachments?.length" class="fc-post-card__atts">
|
||||
<v-icon size="small" class="fc-post-card__att-icon">mdi-paperclip</v-icon>
|
||||
{{ post.attachments.length }} attachment{{ post.attachments.length === 1 ? '' : 's' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="post.thumbnails.length" class="fc-post-card__thumbs">
|
||||
<RouterLink
|
||||
v-for="t in post.thumbnails"
|
||||
:key="t.image_id"
|
||||
:to="{ path: '/gallery', query: { post_id: post.id } }"
|
||||
class="fc-post-card__thumb"
|
||||
>
|
||||
<v-img
|
||||
:src="t.thumbnail_url" cover :alt="`thumbnail`"
|
||||
width="96" height="96" class="fc-post-card__thumb-img"
|
||||
/>
|
||||
</RouterLink>
|
||||
<RouterLink
|
||||
v-if="post.thumbnails_more > 0"
|
||||
:to="{ path: '/gallery', query: { post_id: post.id } }"
|
||||
class="fc-post-card__more-thumbs"
|
||||
>+{{ post.thumbnails_more }} more</RouterLink>
|
||||
</div>
|
||||
<!-- Expanded body: title, full mosaic, full sanitized HTML description,
|
||||
attachments. Lazy-loaded detail via getPostFull. -->
|
||||
<div v-else class="fc-post-card__expanded">
|
||||
<h2 v-if="post.post_title" class="fc-post-card__title-full">
|
||||
{{ post.post_title }}
|
||||
</h2>
|
||||
<h2 v-else class="fc-post-card__title-full fc-post-card__title--missing">
|
||||
Post {{ post.external_post_id }}
|
||||
</h2>
|
||||
|
||||
<div v-if="post.attachments.length" class="fc-post-card__attachments">
|
||||
<a
|
||||
v-for="att in post.attachments"
|
||||
:key="att.id"
|
||||
:href="att.download_url"
|
||||
download
|
||||
class="fc-post-card__att"
|
||||
>
|
||||
<v-icon size="small" class="fc-post-card__att-icon">mdi-paperclip</v-icon>
|
||||
<span class="fc-post-card__att-name">{{ att.original_filename }}</span>
|
||||
<span class="fc-post-card__att-size">({{ formatBytes(att.size_bytes) }})</span>
|
||||
</a>
|
||||
<section v-if="images.length" class="fc-post-card__sec">
|
||||
<PostImageGrid :thumbnails="images" />
|
||||
<div v-if="!detailLoaded" class="fc-post-card__loading-hint">
|
||||
Loading full image list…
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="descriptionHtml" class="fc-post-card__sec">
|
||||
<div class="fc-post-card__desc-full" v-html="descriptionHtml" />
|
||||
</section>
|
||||
<section v-else-if="detailLoaded" class="fc-post-card__sec">
|
||||
<p class="fc-post-card__desc fc-post-card__desc--missing">(no description)</p>
|
||||
</section>
|
||||
|
||||
<section v-if="attachments.length" class="fc-post-card__sec">
|
||||
<h3 class="fc-post-card__h3">Attachments</h3>
|
||||
<div class="fc-post-card__atts-full">
|
||||
<a
|
||||
v-for="att in attachments" :key="att.id"
|
||||
:href="att.download_url" download
|
||||
class="fc-post-card__att"
|
||||
@click.stop
|
||||
>
|
||||
<v-icon size="small" class="fc-post-card__att-icon">mdi-paperclip</v-icon>
|
||||
<span>{{ att.original_filename }}</span>
|
||||
<span class="fc-post-card__att-size">({{ formatBytes(att.size_bytes) }})</span>
|
||||
</a>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
import { RouterLink } from 'vue-router'
|
||||
|
||||
import { usePostsStore } from '../../stores/posts.js'
|
||||
import { sanitizeHtml } from '../../utils/htmlSanitize.js'
|
||||
import PostEmptyThumbs from './PostEmptyThumbs.vue'
|
||||
import PostImageGrid from './PostImageGrid.vue'
|
||||
|
||||
const props = defineProps({
|
||||
post: { type: Object, required: true },
|
||||
@@ -77,8 +135,29 @@ const props = defineProps({
|
||||
|
||||
const postsStore = usePostsStore()
|
||||
|
||||
// Per-card expand state. No global modal — each PostCard owns its own
|
||||
// view-mode and lazy-loaded detail.
|
||||
const expanded = ref(false)
|
||||
const fullDescription = ref(null)
|
||||
const detail = ref(null)
|
||||
const detailLoaded = ref(false)
|
||||
const detailError = ref(null)
|
||||
|
||||
// When expanded + detail loaded, prefer the uncapped detail thumbnails +
|
||||
// full description. Falls back to feed shape if detail fetch is in flight
|
||||
// or failed.
|
||||
const merged = computed(() => detail.value || props.post)
|
||||
const images = computed(() => merged.value.thumbnails || [])
|
||||
const attachments = computed(() => merged.value.attachments || [])
|
||||
|
||||
// Compact-view hero+rail derived from the feed-shape (capped 6).
|
||||
const hero = computed(() => props.post.thumbnails?.[0])
|
||||
const rail = computed(() => (props.post.thumbnails || []).slice(1, 4))
|
||||
const moreCount = computed(() => {
|
||||
const more = props.post.thumbnails_more || 0
|
||||
const railLen = rail.value.length
|
||||
const extraShown = Math.max(0, (props.post.thumbnails?.length || 0) - 1 - railLen)
|
||||
return more + extraShown
|
||||
})
|
||||
|
||||
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
||||
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
||||
@@ -92,20 +171,52 @@ const relativeDate = computed(() => {
|
||||
return new Date(sortDateIso.value).toLocaleDateString()
|
||||
})
|
||||
|
||||
const descriptionToShow = computed(() => {
|
||||
if (expanded.value && fullDescription.value) return fullDescription.value
|
||||
return props.post.description_plain
|
||||
const descriptionHtml = computed(() => {
|
||||
// Detail endpoint returns description_full as plain text (the service
|
||||
// uses html_to_plain on the stored description). Render plain text in
|
||||
// <p> wrappers; sanitize defensively in case the backend ever returns
|
||||
// raw HTML.
|
||||
const raw = merged.value.description_full || merged.value.description_plain
|
||||
if (!raw) return ''
|
||||
if (/[<>]/.test(raw)) return sanitizeHtml(raw)
|
||||
const esc = raw
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
return esc
|
||||
.split(/\n\s*\n/)
|
||||
.map((p) => `<p>${p.replace(/\n/g, '<br>')}</p>`)
|
||||
.join('')
|
||||
})
|
||||
|
||||
async function expand() {
|
||||
if (!fullDescription.value) {
|
||||
const detail = await postsStore.getPostFull(props.post.id)
|
||||
fullDescription.value = detail?.description_full || props.post.description_plain
|
||||
async function loadDetailIfNeeded () {
|
||||
if (detailLoaded.value || detail.value) return
|
||||
try {
|
||||
detail.value = await postsStore.getPostFull(props.post.id)
|
||||
detailLoaded.value = true
|
||||
} catch (e) {
|
||||
detailError.value = e.message
|
||||
// Leave merged on feed-shape; the card still renders the truncated
|
||||
// body so the operator isn't staring at a blank panel.
|
||||
}
|
||||
expanded.value = true
|
||||
}
|
||||
|
||||
function formatBytes(n) {
|
||||
function toggleExpanded () {
|
||||
expanded.value = !expanded.value
|
||||
if (expanded.value) loadDetailIfNeeded()
|
||||
}
|
||||
|
||||
function onCardClick (e) {
|
||||
// Inner interactive elements use @click.stop so they never reach here.
|
||||
// Whole-card click expands a collapsed card; collapsing is chevron-only
|
||||
// so a mosaic-image click on an expanded card can never accidentally
|
||||
// collapse the surrounding card.
|
||||
if (expanded.value) return
|
||||
expanded.value = true
|
||||
loadDetailIfNeeded()
|
||||
}
|
||||
|
||||
function formatBytes (n) {
|
||||
if (!n) return '0 B'
|
||||
if (n < 1024) return `${n} B`
|
||||
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
|
||||
@@ -118,13 +229,31 @@ function formatBytes(n) {
|
||||
.fc-post-card {
|
||||
padding: 1rem;
|
||||
margin-bottom: 1rem;
|
||||
container-type: inline-size;
|
||||
transition: border-color 0.15s ease;
|
||||
}
|
||||
.fc-post-card:not(.fc-post-card--expanded) {
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-post-card:not(.fc-post-card--expanded):hover {
|
||||
border-color: rgb(var(--v-theme-accent));
|
||||
}
|
||||
.fc-post-card:focus-visible {
|
||||
outline: 2px solid rgb(var(--v-theme-accent));
|
||||
outline-offset: 2px;
|
||||
}
|
||||
.fc-post-card--expanded {
|
||||
border-color: rgb(var(--v-theme-accent) / 0.6);
|
||||
}
|
||||
|
||||
.fc-post-card__head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.6rem;
|
||||
font-size: 0.85rem;
|
||||
font-size: 0.8rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin-bottom: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.fc-post-card__artist {
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
@@ -132,69 +261,154 @@ function formatBytes(n) {
|
||||
font-weight: 600;
|
||||
}
|
||||
.fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||
.fc-post-card__date { white-space: nowrap; }
|
||||
.fc-post-card__title {
|
||||
font-size: 1.05rem;
|
||||
margin: 0.6rem 0 0.4rem;
|
||||
}
|
||||
.fc-post-card__desc {
|
||||
white-space: pre-wrap;
|
||||
font-size: 0.92rem;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin-bottom: 0.6rem;
|
||||
}
|
||||
.fc-post-card__more {
|
||||
background: none;
|
||||
border: none;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
cursor: pointer;
|
||||
padding: 0 0.25rem;
|
||||
font-size: inherit;
|
||||
}
|
||||
.fc-post-card__thumbs {
|
||||
.fc-post-card__date,
|
||||
.fc-post-card__meta { white-space: nowrap; }
|
||||
|
||||
/* ---- COMPACT BODY ---- */
|
||||
.fc-post-card__body {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.4rem;
|
||||
align-items: center;
|
||||
margin: 0.5rem 0;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
.fc-post-card__thumb { line-height: 0; }
|
||||
.fc-post-card__thumb-img {
|
||||
border-radius: 4px;
|
||||
@container (min-width: 800px) {
|
||||
.fc-post-card__body {
|
||||
flex-direction: row;
|
||||
gap: 24px;
|
||||
}
|
||||
.fc-post-card__media { flex: 0 0 50%; }
|
||||
.fc-post-card__text { flex: 1 1 0; min-width: 0; }
|
||||
}
|
||||
|
||||
.fc-post-card__hero {
|
||||
width: 100%;
|
||||
aspect-ratio: 16 / 10;
|
||||
overflow: hidden;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.fc-post-card__more-thumbs {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 96px;
|
||||
height: 96px;
|
||||
.fc-post-card__hero img {
|
||||
width: 100%; height: 100%;
|
||||
object-fit: cover; display: block;
|
||||
}
|
||||
|
||||
.fc-post-card__rail {
|
||||
display: flex; gap: 6px; margin-top: 6px;
|
||||
}
|
||||
.fc-post-card__rail-cell {
|
||||
width: 80px; height: 80px;
|
||||
overflow: hidden; border-radius: 4px;
|
||||
}
|
||||
.fc-post-card__rail-cell img {
|
||||
width: 100%; height: 100%;
|
||||
object-fit: cover; display: block;
|
||||
}
|
||||
.fc-post-card__rail-more {
|
||||
width: 80px; height: 80px;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
border: 1px dashed rgb(var(--v-theme-on-surface-variant));
|
||||
border-radius: 4px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
text-decoration: none;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.fc-post-card__more-thumbs:hover {
|
||||
color: rgb(var(--v-theme-accent));
|
||||
border-color: rgb(var(--v-theme-accent));
|
||||
|
||||
.fc-post-card__title {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 18px; font-weight: 500;
|
||||
margin: 0 0 8px 0;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 2;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fc-post-card__attachments {
|
||||
.fc-post-card__title--missing {
|
||||
font-style: italic;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
@container (min-width: 800px) {
|
||||
.fc-post-card__title {
|
||||
font-size: 20px;
|
||||
-webkit-line-clamp: 1;
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
}
|
||||
|
||||
.fc-post-card__desc {
|
||||
font-size: 0.9rem;
|
||||
line-height: 1.5;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin: 0 0 12px 0;
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 3;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fc-post-card__desc--missing {
|
||||
font-style: italic;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
@container (min-width: 800px) {
|
||||
.fc-post-card__desc { -webkit-line-clamp: 5; }
|
||||
}
|
||||
|
||||
.fc-post-card__atts {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 0.85rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-post-card__att-icon { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
|
||||
/* ---- EXPANDED BODY ---- */
|
||||
.fc-post-card__expanded {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.4rem;
|
||||
margin-top: 0.6rem;
|
||||
flex-direction: column;
|
||||
gap: 20px;
|
||||
}
|
||||
.fc-post-card__title-full {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 22px;
|
||||
font-weight: 500;
|
||||
margin: 0;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
@container (min-width: 800px) {
|
||||
.fc-post-card__title-full { font-size: 26px; }
|
||||
}
|
||||
.fc-post-card__sec { margin: 0; }
|
||||
.fc-post-card__h3 {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 16px;
|
||||
font-weight: 500;
|
||||
margin: 0 0 8px 0;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-post-card__loading-hint {
|
||||
margin-top: 8px;
|
||||
font-size: 0.8rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-post-card__desc-full {
|
||||
font-size: 0.95rem;
|
||||
line-height: 1.55;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-post-card__desc-full :deep(p) { margin: 0 0 12px 0; }
|
||||
.fc-post-card__desc-full :deep(a) { color: rgb(var(--v-theme-accent)); }
|
||||
.fc-post-card__atts-full {
|
||||
display: flex; flex-wrap: wrap; gap: 8px;
|
||||
}
|
||||
.fc-post-card__att {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.35rem;
|
||||
padding: 0.2rem 0.5rem;
|
||||
gap: 6px;
|
||||
padding: 6px 12px;
|
||||
border: 1px solid rgb(var(--v-theme-on-surface-variant));
|
||||
border-radius: 999px;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
text-decoration: none;
|
||||
font-size: 0.82rem;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.fc-post-card__att:hover {
|
||||
color: rgb(var(--v-theme-accent));
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
<template>
|
||||
<div class="fc-post-empty">
|
||||
<v-icon size="48" class="fc-post-empty__icon">mdi-image-off-outline</v-icon>
|
||||
<div class="fc-post-empty__text">No images attached to this post</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
// No props — pure presentational placeholder.
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-post-empty {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 12px;
|
||||
padding: 32px 16px;
|
||||
border: 1px dashed rgb(var(--v-theme-on-surface-variant));
|
||||
border-radius: 8px;
|
||||
background: rgba(20, 23, 26, 0.3);
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 200px;
|
||||
}
|
||||
.fc-post-empty__icon {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
opacity: 0.6;
|
||||
}
|
||||
.fc-post-empty__text {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,63 @@
|
||||
<template>
|
||||
<div class="fc-post-grid">
|
||||
<button
|
||||
v-for="(t, idx) in thumbnails"
|
||||
:key="t.image_id"
|
||||
type="button"
|
||||
class="fc-post-grid__cell"
|
||||
:aria-label="`Open image ${idx + 1} of ${thumbnails.length}`"
|
||||
@click="openImage(t.image_id, idx)"
|
||||
>
|
||||
<img
|
||||
:src="t.thumbnail_url"
|
||||
:alt="`thumbnail ${idx + 1}`"
|
||||
loading="lazy"
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { useModalStore } from '../../stores/modal.js'
|
||||
|
||||
const props = defineProps({
|
||||
thumbnails: { type: Array, required: true }, // [{ image_id, thumbnail_url, ... }]
|
||||
})
|
||||
|
||||
const modal = useModalStore()
|
||||
|
||||
const imageIds = computed(() => props.thumbnails.map(t => t.image_id))
|
||||
|
||||
function openImage (id, idx) {
|
||||
modal.open(id, { postImageIds: imageIds.value, initialIndex: idx })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-post-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
|
||||
gap: 6px;
|
||||
}
|
||||
.fc-post-grid__cell {
|
||||
aspect-ratio: 4 / 3;
|
||||
overflow: hidden;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
background: rgb(var(--v-theme-background));
|
||||
transition: transform 0.15s ease, box-shadow 0.15s ease;
|
||||
}
|
||||
.fc-post-grid__cell:hover {
|
||||
transform: scale(1.02);
|
||||
box-shadow: 0 0 0 2px rgb(var(--v-theme-accent));
|
||||
}
|
||||
.fc-post-grid__cell img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
}
|
||||
</style>
|
||||
@@ -34,11 +34,18 @@
|
||||
|
||||
<template v-else-if="manifest?.installed">
|
||||
<div class="fc-ext-install mt-3">
|
||||
<!-- Install button: direct :href anchor click (no programmatic
|
||||
window.location.assign). Firefox's XPI-install gesture
|
||||
requires a user-clicked anchor pointing at an
|
||||
application/x-xpinstall response; programmatic navigation
|
||||
sometimes triggered nothing instead of the install dialog
|
||||
(operator-flagged 2026-05-26). No `download` attribute —
|
||||
that would force a save dialog instead of install. -->
|
||||
<v-btn
|
||||
v-if="isFirefox"
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-firefox"
|
||||
@click="installXpi"
|
||||
:href="manifest.latest_url"
|
||||
>Install Firefox extension</v-btn>
|
||||
|
||||
<v-btn
|
||||
@@ -103,6 +110,7 @@
|
||||
<script setup>
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { copyText } from '../../utils/clipboard.js'
|
||||
|
||||
const api = useApi()
|
||||
|
||||
@@ -146,11 +154,6 @@ async function loadKey() {
|
||||
}
|
||||
}
|
||||
|
||||
function installXpi() {
|
||||
if (!manifest.value?.latest_url) return
|
||||
window.location.assign(manifest.value.latest_url)
|
||||
}
|
||||
|
||||
async function rotateKey() {
|
||||
rotating.value = true
|
||||
try {
|
||||
@@ -170,7 +173,7 @@ async function rotateKey() {
|
||||
|
||||
async function copy(text, label) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text)
|
||||
await copyText(text)
|
||||
window.__fcToast?.({ text: `${label} copied.`, type: 'success' })
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||
|
||||
@@ -51,68 +51,6 @@
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<v-divider class="my-4" />
|
||||
<div class="text-subtitle-2 mb-2">Downloader (FC-3c)</div>
|
||||
<v-row>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field
|
||||
v-model.number="local.download_rate_limit_seconds"
|
||||
label="Rate limit (seconds between requests)"
|
||||
type="number" step="0.5" min="0"
|
||||
density="compact" hide-details @blur="save"
|
||||
/>
|
||||
<div class="fc-help">gallery-dl extractor.sleep. Higher = slower but safer.</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-switch
|
||||
v-model="local.download_validate_files"
|
||||
label="Validate downloaded files (magic-byte check)"
|
||||
density="compact" hide-details color="accent" @change="save"
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<v-divider class="my-4" />
|
||||
<div class="text-subtitle-2 mb-2">Download scheduling (FC-3d)</div>
|
||||
<v-row>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field
|
||||
v-model.number="local.download_schedule_default_seconds"
|
||||
label="Default check interval (seconds)"
|
||||
type="number" :min="60" :max="86400"
|
||||
density="compact" hide-details @blur="save"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Used when a source has no per-source or per-artist override.
|
||||
Default 28800 (8 hours).
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field
|
||||
v-model.number="local.download_event_retention_days"
|
||||
label="Event retention (days)"
|
||||
type="number" :min="1" :max="3650"
|
||||
density="compact" hide-details @blur="save"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Completed download events older than this are deleted nightly.
|
||||
Default 90.
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field
|
||||
v-model.number="local.download_failure_warning_threshold"
|
||||
label="Failure warning threshold"
|
||||
type="number" :min="1" :max="100"
|
||||
density="compact" hide-details @blur="save"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Source row badge turns red after this many consecutive
|
||||
failures. Sources are never auto-disabled. Default 5.
|
||||
</div>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<v-alert v-if="store.settingsError" type="error" variant="tonal" class="mt-2" closable>
|
||||
{{ store.settingsError }}
|
||||
</v-alert>
|
||||
@@ -128,16 +66,14 @@ import { reactive, watch } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
// Downloader + schedule-defaults fields moved to
|
||||
// /subscriptions?tab=settings (operator decision 2026-05-27). This form
|
||||
// now only owns image-import filters.
|
||||
const local = reactive({
|
||||
min_width: 0, min_height: 0,
|
||||
skip_transparent: false, transparency_threshold: 0.9,
|
||||
skip_single_color: false, single_color_threshold: 0.95,
|
||||
phash_threshold: 10,
|
||||
download_rate_limit_seconds: 3.0,
|
||||
download_validate_files: true,
|
||||
download_schedule_default_seconds: 28800,
|
||||
download_event_retention_days: 90,
|
||||
download_failure_warning_threshold: 5,
|
||||
})
|
||||
|
||||
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||
|
||||
@@ -45,9 +45,24 @@
|
||||
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
||||
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
||||
<template #item.error="{ item }">
|
||||
<span v-if="item.error" :title="item.error" class="text-caption">
|
||||
{{ shorten(item.error, 60) }}
|
||||
</span>
|
||||
<button
|
||||
v-if="item.error" type="button" class="fc-err-link text-caption"
|
||||
@click="openError(`Task ${item.id} failed`, item.error)"
|
||||
title="Click for full error"
|
||||
>{{ shorten(item.error, 60) }}</button>
|
||||
</template>
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
v-if="item.status === 'failed'"
|
||||
icon size="x-small" variant="text"
|
||||
:loading="refetching === item.id"
|
||||
@click="onRefetch(item)"
|
||||
>
|
||||
<v-icon size="small">mdi-cloud-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">
|
||||
Re-fetch original (re-download from source)
|
||||
</v-tooltip>
|
||||
</v-btn>
|
||||
</template>
|
||||
</v-data-table-virtual>
|
||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||
@@ -100,16 +115,35 @@
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<ErrorDetailModal
|
||||
v-model="showErrorModal"
|
||||
:title="errorModalTitle"
|
||||
:message="errorModalMessage"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
|
||||
const store = useImportStore()
|
||||
const statusFilter = ref(null)
|
||||
const clearDialog = ref(false)
|
||||
// Click-to-open modal for full error text (operator-flagged 2026-05-26
|
||||
// — the prior :title="..." tooltip cramped multi-line SQLAlchemy
|
||||
// tracebacks into an unusable popup with no copy-paste affordance).
|
||||
const showErrorModal = ref(false)
|
||||
const errorModalTitle = ref('')
|
||||
const errorModalMessage = ref('')
|
||||
|
||||
function openError(title, message) {
|
||||
errorModalTitle.value = title
|
||||
errorModalMessage.value = message || ''
|
||||
showErrorModal.value = true
|
||||
}
|
||||
const clearAgeDays = ref(7)
|
||||
const clearStuckDialog = ref(false)
|
||||
|
||||
@@ -128,9 +162,29 @@ const headers = [
|
||||
{ title: 'Source', key: 'source_path', sortable: false },
|
||||
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
||||
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
||||
{ title: 'Note', key: 'error', sortable: false }
|
||||
{ title: 'Note', key: 'error', sortable: false },
|
||||
{ title: '', key: 'actions', sortable: false, width: 56 }
|
||||
]
|
||||
|
||||
const refetching = ref(null)
|
||||
const _REFETCH_MSG = {
|
||||
refetch_queued: { text: 'Re-fetch queued — re-downloading from source', type: 'success' },
|
||||
no_source: { text: 'No re-fetchable source (filesystem import — replace the file manually)', type: 'info' },
|
||||
already_refetched: { text: 'Already re-fetched once', type: 'info' },
|
||||
}
|
||||
async function onRefetch(item) {
|
||||
refetching.value = item.id
|
||||
try {
|
||||
const res = await store.refetchTask(item.id)
|
||||
const msg = _REFETCH_MSG[res.status] || { text: `Re-fetch: ${res.status}`, type: 'info' }
|
||||
window.__fcToast?.(msg)
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Re-fetch failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
refetching.value = null
|
||||
}
|
||||
}
|
||||
|
||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||
const hasStuck = computed(() => store.tasks.some(
|
||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||
@@ -179,3 +233,21 @@ async function onClearStuckConfirm() {
|
||||
clearStuckDialog.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-err-link {
|
||||
/* Truncated error preview as a clickable button — opens
|
||||
ErrorDetailModal with the full text. Inherits the row's font
|
||||
sizing so it doesn't visually drift from the prior tooltip-bearing
|
||||
span. */
|
||||
color: rgb(var(--v-theme-error, 220 80 80));
|
||||
background: transparent;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
font: inherit;
|
||||
text-align: left;
|
||||
text-decoration: underline dotted;
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-err-link:hover { text-decoration: underline; }
|
||||
</style>
|
||||
|
||||
@@ -7,8 +7,12 @@
|
||||
indeterminate color="accent" size="20"
|
||||
/>
|
||||
<span>
|
||||
Scanning {{ store.activeBatch.source_path }} —
|
||||
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
|
||||
{{ store.activeBatch.source_path || '/import' }} —
|
||||
imported {{ store.activeBatch.imported }},
|
||||
<template v-if="store.activeBatch.scan_mode === 'deep'">
|
||||
refreshed {{ store.activeBatch.refreshed || 0 }},
|
||||
</template>
|
||||
skipped {{ store.activeBatch.skipped }},
|
||||
failed {{ store.activeBatch.failed }} /
|
||||
{{ store.activeBatch.total_files }} files
|
||||
@@ -24,8 +28,13 @@
|
||||
|
||||
<p class="text-body-2 mb-3">
|
||||
<span v-if="!store.activeBatch">
|
||||
Run a quick scan of the import directory. Deep scan (pHash dedup,
|
||||
archives) lands in FC-2d.
|
||||
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
|
||||
new files only.
|
||||
<strong>Deep scan</strong> additionally re-walks already-imported
|
||||
files so updated sidecar metadata (post title/date/attribution) and
|
||||
previously-NULL phashes / artist links get refreshed. Use after
|
||||
bulk-downloading fresh sidecars for existing content. Both modes
|
||||
route non-media + sidecar pairs through PostAttachment capture.
|
||||
</span>
|
||||
<span v-else>
|
||||
An active batch is in progress. Wait for it to finish, or click
|
||||
@@ -34,15 +43,26 @@
|
||||
</span>
|
||||
</p>
|
||||
|
||||
<v-btn
|
||||
color="primary" rounded="pill"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy"
|
||||
@click="trigger"
|
||||
>
|
||||
<v-icon start>mdi-magnify-scan</v-icon>
|
||||
Quick scan
|
||||
</v-btn>
|
||||
<div class="d-flex flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" rounded="pill"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'quick'"
|
||||
@click="trigger('quick')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-scan</v-icon>
|
||||
Quick scan
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="secondary" rounded="pill" variant="tonal"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'deep'"
|
||||
@click="trigger('deep')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-plus-outline</v-icon>
|
||||
Deep scan
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
|
||||
{{ store.triggerError }}
|
||||
@@ -56,12 +76,12 @@ import { ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
const busy = ref(false)
|
||||
const busy = ref(null)
|
||||
const clearing = ref(false)
|
||||
|
||||
async function trigger() {
|
||||
busy.value = true
|
||||
try { await store.triggerScan() } catch {} finally { busy.value = false }
|
||||
async function trigger(mode) {
|
||||
busy.value = mode
|
||||
try { await store.triggerScan(mode) } catch {} finally { busy.value = null }
|
||||
}
|
||||
|
||||
async function onClearStuck() {
|
||||
|
||||
@@ -1,20 +1,23 @@
|
||||
<template>
|
||||
<div class="fc-maint">
|
||||
<p class="text-body-2 mb-4">
|
||||
Machine-assisted tagging controls. Backfill and centroid recompute run
|
||||
nightly automatically; the allowlist auto-applies accepted tags to new
|
||||
and existing images.
|
||||
Operational backfills and tagging controls. The ML backfill and centroid
|
||||
recompute run nightly automatically; the allowlist auto-applies accepted
|
||||
tags to new and existing images. Use the cards below to trigger a
|
||||
one-off pass.
|
||||
</p>
|
||||
<div class="fc-maint__grid">
|
||||
<MLBackfillCard />
|
||||
<CentroidRecomputeCard />
|
||||
<ThumbnailBackfillCard />
|
||||
</div>
|
||||
<MLThresholdSliders class="mt-4" />
|
||||
<AllowlistTable class="mt-4" />
|
||||
<AliasTable class="mt-4" />
|
||||
<BackupCard class="mt-6" />
|
||||
<TagMaintenanceCard class="mt-6" />
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||
operates on the existing library which fits the Cleanup-tab
|
||||
theme, and clusters with the other audit cards. -->
|
||||
<LegacyMigrationCard class="mt-6" />
|
||||
</div>
|
||||
</template>
|
||||
@@ -22,12 +25,11 @@
|
||||
<script setup>
|
||||
import MLBackfillCard from './MLBackfillCard.vue'
|
||||
import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||
import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||
import AllowlistTable from './AllowlistTable.vue'
|
||||
import AliasTable from './AliasTable.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import TagMaintenanceCard from './TagMaintenanceCard.vue'
|
||||
import BrowserExtensionCard from './BrowserExtensionCard.vue'
|
||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
||||
</script>
|
||||
|
||||
|
||||
@@ -59,8 +59,12 @@
|
||||
<td>{{ r.queue }}</td>
|
||||
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||
<td class="fc-err" :title="r.error_message">
|
||||
{{ r.error_type }}
|
||||
<td>
|
||||
<button
|
||||
type="button" class="fc-err-link"
|
||||
@click="openError(r)"
|
||||
:title="'Click for full error'"
|
||||
>{{ r.error_type }}</button>
|
||||
</td>
|
||||
</tr>
|
||||
<tr v-if="!filteredFailures.length">
|
||||
@@ -138,6 +142,11 @@
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<ErrorDetailModal
|
||||
v-model="showErrorModal"
|
||||
:row="errorModalRow"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -145,8 +154,21 @@
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
import QueuesTable from './QueuesTable.vue'
|
||||
|
||||
// Click-to-open modal for full error text. Replaces the unusable
|
||||
// :title="..." tooltip (operator-flagged 2026-05-26: SQLAlchemy
|
||||
// rollback + traceback content rendered as a cramped browser tooltip
|
||||
// you couldn't copy from or scroll within).
|
||||
const showErrorModal = ref(false)
|
||||
const errorModalRow = ref(null)
|
||||
|
||||
function openError(row) {
|
||||
errorModalRow.value = row
|
||||
showErrorModal.value = true
|
||||
}
|
||||
|
||||
const store = useSystemActivityStore()
|
||||
|
||||
const filterQueue = ref(null)
|
||||
@@ -276,5 +298,18 @@ function formatRelative(iso) {
|
||||
font-feature-settings: 'tnum';
|
||||
}
|
||||
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||
.fc-err-link {
|
||||
/* Styled as a text-only button so the error_type cell stays
|
||||
visually identical to the prior tooltip-bearing row, but is
|
||||
now a real clickable target with hover affordance. */
|
||||
color: rgb(var(--v-theme-error, 220 80 80));
|
||||
background: transparent;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
font: inherit;
|
||||
text-decoration: underline dotted;
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-err-link:hover { text-decoration: underline; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<v-card-title>Thumbnail backfill</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 mb-3">
|
||||
Scan the library for images with no thumbnail, or whose thumbnail file
|
||||
is missing or corrupt on disk. Repair candidates are re-enqueued for
|
||||
thumbnail generation. Safe to re-run.
|
||||
</p>
|
||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
||||
</v-btn>
|
||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
||||
const store = useThumbnailsStore()
|
||||
const busy = ref(false)
|
||||
const done = ref(false)
|
||||
async function run () {
|
||||
busy.value = true
|
||||
try { await store.triggerBackfill(); done.value = true }
|
||||
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
||||
finally { busy.value = false }
|
||||
}
|
||||
</script>
|
||||
@@ -0,0 +1,148 @@
|
||||
<template>
|
||||
<v-card
|
||||
:variant="hasCredential ? 'outlined' : 'flat'"
|
||||
:class="['fc-cred-card', hasCredential && 'fc-cred-card--set']"
|
||||
>
|
||||
<v-card-title class="d-flex align-center pa-3 ga-2">
|
||||
<PlatformChip :platform="platform.key" size="small" />
|
||||
<span class="text-body-1">{{ platform.name }}</span>
|
||||
<v-spacer />
|
||||
<v-chip :color="statusColor" size="x-small" variant="tonal">
|
||||
{{ statusLabel }}
|
||||
</v-chip>
|
||||
</v-card-title>
|
||||
|
||||
<v-card-text class="pa-3 pt-0">
|
||||
<template v-if="hasCredential">
|
||||
<div class="fc-cred-card__row">
|
||||
<v-icon size="small" color="success">mdi-check-circle</v-icon>
|
||||
<span>Stored · captured {{ fmtDate(credential.captured_at) }}</span>
|
||||
</div>
|
||||
<div v-if="credential.expires_at" class="fc-cred-card__row">
|
||||
<v-icon size="small" :color="expiringSoon ? 'warning' : 'on-surface-variant'">
|
||||
mdi-clock-outline
|
||||
</v-icon>
|
||||
<span>Expires {{ fmtDate(credential.expires_at) }}</span>
|
||||
</div>
|
||||
<div v-if="credential.last_verified_at" class="fc-cred-card__row">
|
||||
<v-icon size="small" color="on-surface-variant">mdi-shield-check</v-icon>
|
||||
<span>Last verified {{ fmtDate(credential.last_verified_at) }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else>
|
||||
<div class="fc-cred-card__empty">
|
||||
<v-icon size="40" color="on-surface-variant" class="fc-cred-card__empty-icon">
|
||||
mdi-key-remove
|
||||
</v-icon>
|
||||
<p class="text-caption text-medium-emphasis">
|
||||
No credential stored. Use the extension or paste a {{ platform.auth_type }} below.
|
||||
</p>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<v-expansion-panels variant="accordion" class="mt-2 fc-cred-card__how">
|
||||
<v-expansion-panel>
|
||||
<v-expansion-panel-title class="text-caption">
|
||||
How to get {{ platform.auth_type }}
|
||||
</v-expansion-panel-title>
|
||||
<v-expansion-panel-text class="text-caption">
|
||||
<slot name="howto">
|
||||
<p>{{ howToFallback }}</p>
|
||||
</slot>
|
||||
</v-expansion-panel-text>
|
||||
</v-expansion-panel>
|
||||
</v-expansion-panels>
|
||||
</v-card-text>
|
||||
|
||||
<v-card-actions class="px-3 pb-3 pt-0">
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
v-if="hasCredential"
|
||||
size="small" variant="text" color="error"
|
||||
@click="$emit('remove', platform)"
|
||||
>
|
||||
Remove
|
||||
</v-btn>
|
||||
<v-btn
|
||||
size="small"
|
||||
:variant="hasCredential ? 'outlined' : 'flat'"
|
||||
:color="hasCredential ? undefined : 'accent'"
|
||||
@click="$emit('replace', platform)"
|
||||
>
|
||||
{{ hasCredential ? 'Update' : 'Add credentials' }}
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import PlatformChip from './PlatformChip.vue'
|
||||
|
||||
const props = defineProps({
|
||||
platform: { type: Object, required: true },
|
||||
credential: { type: Object, default: null },
|
||||
})
|
||||
defineEmits(['replace', 'remove'])
|
||||
|
||||
const hasCredential = computed(() => !!props.credential)
|
||||
|
||||
// Within 7 days = expiring soon. The backend doesn't set hard rotation
|
||||
// policy yet; this just nudges the operator with a warning chip.
|
||||
const expiringSoon = computed(() => {
|
||||
const exp = props.credential?.expires_at
|
||||
if (!exp) return false
|
||||
const diff = (new Date(exp).getTime() - Date.now()) / 86400_000
|
||||
return diff > 0 && diff < 7
|
||||
})
|
||||
|
||||
const statusLabel = computed(() => {
|
||||
if (!hasCredential.value) return 'Not configured'
|
||||
if (expiringSoon.value) return 'Expiring soon'
|
||||
return 'Active'
|
||||
})
|
||||
const statusColor = computed(() => {
|
||||
if (!hasCredential.value) return 'grey'
|
||||
if (expiringSoon.value) return 'warning'
|
||||
return 'success'
|
||||
})
|
||||
|
||||
const howToFallback = computed(() => {
|
||||
if (props.platform.auth_type === 'cookies') {
|
||||
return 'Use the FabledCurator browser extension on the platform page, or export cookies.txt and paste here.'
|
||||
}
|
||||
return 'Paste the access token from your account settings.'
|
||||
})
|
||||
|
||||
function fmtDate(iso) {
|
||||
if (!iso) return '—'
|
||||
return iso.slice(0, 10)
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-cred-card {
|
||||
border: 1px dashed rgb(var(--v-theme-on-surface-variant) / 0.3);
|
||||
background: rgb(var(--v-theme-surface));
|
||||
height: 100%;
|
||||
}
|
||||
.fc-cred-card--set {
|
||||
border-style: solid;
|
||||
border-color: rgb(var(--v-theme-accent) / 0.5);
|
||||
}
|
||||
.fc-cred-card__row {
|
||||
display: flex; gap: 6px; align-items: center;
|
||||
font-size: 0.9rem;
|
||||
margin-top: 4px;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-cred-card__empty {
|
||||
display: flex; flex-direction: column; align-items: center; gap: 8px;
|
||||
padding: 12px 0;
|
||||
text-align: center;
|
||||
}
|
||||
.fc-cred-card__empty-icon { opacity: 0.5; }
|
||||
.fc-cred-card__how :deep(.v-expansion-panel) {
|
||||
background: transparent;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,36 @@
|
||||
<template>
|
||||
<div class="fc-dl-stats">
|
||||
<v-chip
|
||||
v-for="s in STAT_DEFS" :key="s.key"
|
||||
:color="s.color"
|
||||
variant="tonal"
|
||||
:prepend-icon="s.icon"
|
||||
size="default"
|
||||
>
|
||||
{{ s.label }}
|
||||
<strong class="ms-1">{{ stats[s.key] ?? 0 }}</strong>
|
||||
</v-chip>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
defineProps({
|
||||
stats: { type: Object, required: true },
|
||||
})
|
||||
|
||||
// status keys come straight from the backend ENUM
|
||||
// (pending|running|ok|error|skipped); display order + icons are UI-only.
|
||||
const STAT_DEFS = [
|
||||
{ key: 'pending', label: 'Queued', color: 'grey', icon: 'mdi-clock-outline' },
|
||||
{ key: 'running', label: 'Running', color: 'info', icon: 'mdi-progress-clock' },
|
||||
{ key: 'ok', label: 'Completed', color: 'success', icon: 'mdi-check-circle' },
|
||||
{ key: 'error', label: 'Failed', color: 'error', icon: 'mdi-alert-circle' },
|
||||
{ key: 'skipped', label: 'Skipped', color: 'warning', icon: 'mdi-skip-next' },
|
||||
]
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dl-stats {
|
||||
display: flex; gap: 8px; flex-wrap: wrap;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,120 @@
|
||||
<template>
|
||||
<div class="fc-dlf">
|
||||
<v-menu :close-on-content-click="false" v-model="open">
|
||||
<template #activator="{ props }">
|
||||
<v-btn v-bind="props" variant="outlined" prepend-icon="mdi-filter-variant">
|
||||
Filter
|
||||
<v-chip v-if="activeCount" size="x-small" color="accent" class="ms-2">
|
||||
{{ activeCount }}
|
||||
</v-chip>
|
||||
</v-btn>
|
||||
</template>
|
||||
<v-card min-width="320" class="pa-3">
|
||||
<v-select
|
||||
v-model="local.status"
|
||||
:items="STATUS_OPTIONS"
|
||||
label="Status"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
class="mb-2"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model.number="local.source_id"
|
||||
label="Source ID"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
type="number" min="1"
|
||||
class="mb-2"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="local.from_date"
|
||||
label="From"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
type="date"
|
||||
class="mb-2"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="local.to_date"
|
||||
label="To"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
type="date"
|
||||
class="mb-3"
|
||||
/>
|
||||
<div class="d-flex">
|
||||
<v-btn variant="text" size="small" @click="reset">Reset</v-btn>
|
||||
<v-spacer />
|
||||
<v-btn color="accent" size="small" @click="apply">Apply</v-btn>
|
||||
</div>
|
||||
</v-card>
|
||||
</v-menu>
|
||||
|
||||
<div v-if="activeCount" class="fc-dlf__pills mt-2">
|
||||
<v-chip
|
||||
v-for="p in activePills" :key="p.key"
|
||||
size="small" closable variant="tonal"
|
||||
@click:close="clearOne(p.key)"
|
||||
>
|
||||
{{ p.label }}
|
||||
</v-chip>
|
||||
<v-btn variant="text" size="x-small" @click="reset">Clear all</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, reactive, ref, watch } from 'vue'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Object, required: true },
|
||||
})
|
||||
const emit = defineEmits(['update:modelValue'])
|
||||
|
||||
const STATUS_OPTIONS = [
|
||||
{ title: 'Queued', value: 'pending' },
|
||||
{ title: 'Running', value: 'running' },
|
||||
{ title: 'Completed', value: 'ok' },
|
||||
{ title: 'Failed', value: 'error' },
|
||||
{ title: 'Skipped', value: 'skipped' },
|
||||
]
|
||||
const STATUS_LABEL = Object.fromEntries(STATUS_OPTIONS.map((o) => [o.value, o.title]))
|
||||
|
||||
const open = ref(false)
|
||||
const local = reactive({
|
||||
status: props.modelValue.status ?? null,
|
||||
source_id: props.modelValue.source_id ?? null,
|
||||
from_date: props.modelValue.from_date ?? null,
|
||||
to_date: props.modelValue.to_date ?? null,
|
||||
})
|
||||
|
||||
watch(() => props.modelValue, (v) => Object.assign(local, v), { deep: true })
|
||||
|
||||
const activePills = computed(() => {
|
||||
const out = []
|
||||
if (local.status) out.push({ key: 'status', label: `Status: ${STATUS_LABEL[local.status] || local.status}` })
|
||||
if (local.source_id) out.push({ key: 'source_id', label: `Source #${local.source_id}` })
|
||||
if (local.from_date) out.push({ key: 'from_date', label: `From ${local.from_date}` })
|
||||
if (local.to_date) out.push({ key: 'to_date', label: `To ${local.to_date}` })
|
||||
return out
|
||||
})
|
||||
const activeCount = computed(() => activePills.value.length)
|
||||
|
||||
function apply() {
|
||||
emit('update:modelValue', { ...local })
|
||||
open.value = false
|
||||
}
|
||||
function reset() {
|
||||
local.status = null
|
||||
local.source_id = null
|
||||
local.from_date = null
|
||||
local.to_date = null
|
||||
emit('update:modelValue', { ...local })
|
||||
}
|
||||
function clearOne(key) {
|
||||
local[key] = null
|
||||
emit('update:modelValue', { ...local })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dlf__pills {
|
||||
display: flex; flex-wrap: wrap; gap: 6px; align-items: center;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,229 @@
|
||||
<template>
|
||||
<div>
|
||||
<div class="fc-dl__top">
|
||||
<DownloadStatChips :stats="store.stats" />
|
||||
<v-spacer />
|
||||
<v-btn variant="text" icon @click="refresh">
|
||||
<v-icon>mdi-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Refresh</v-tooltip>
|
||||
</v-btn>
|
||||
<MaintenanceMenu @refresh="refresh" />
|
||||
</div>
|
||||
|
||||
<DownloadsFilterPopover v-model="filterModel" class="fc-dl__filter" />
|
||||
|
||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="my-4">
|
||||
{{ String(store.error) }}
|
||||
</v-alert>
|
||||
|
||||
<div v-if="store.loading && store.events.length === 0" class="fc-dl__loading">
|
||||
<v-progress-circular indeterminate color="accent" size="36" />
|
||||
</div>
|
||||
|
||||
<div v-else-if="filteredEvents.length === 0" class="fc-dl__empty">
|
||||
<p>No download events match the current filter.</p>
|
||||
</div>
|
||||
|
||||
<div v-else>
|
||||
<section
|
||||
v-for="g in groups" :key="g.key"
|
||||
class="fc-dl__group"
|
||||
>
|
||||
<header
|
||||
class="fc-dl__group-head"
|
||||
role="button" tabindex="0"
|
||||
@click="toggle(g.key)" @keydown.enter="toggle(g.key)"
|
||||
>
|
||||
<v-icon size="small" class="fc-dl__group-chev">
|
||||
{{ collapsed[g.key] ? 'mdi-chevron-right' : 'mdi-chevron-down' }}
|
||||
</v-icon>
|
||||
<span class="fc-dl__group-label">{{ g.label }}</span>
|
||||
<span class="fc-dl__group-counts">
|
||||
<v-chip
|
||||
v-if="g.failedCount > 0"
|
||||
size="x-small" color="error" variant="tonal"
|
||||
prepend-icon="mdi-alert-circle"
|
||||
>{{ g.failedCount }}</v-chip>
|
||||
<v-chip size="x-small" variant="tonal">
|
||||
{{ g.items.length }}
|
||||
{{ g.items.length === 1 ? 'event' : 'events' }}
|
||||
</v-chip>
|
||||
</span>
|
||||
</header>
|
||||
<div v-if="!collapsed[g.key]" class="fc-dl__group-body">
|
||||
<DownloadEventRow
|
||||
v-for="e in g.items" :key="e.id" :event="e"
|
||||
@open="openDetail"
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="fc-dl__sentinel">
|
||||
<v-btn v-if="store.hasMore" variant="text" @click="store.loadMore()" :loading="store.loading">
|
||||
Load more
|
||||
</v-btn>
|
||||
<span v-else class="text-caption" style="opacity: 0.5">No more events.</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<DownloadDetailModal
|
||||
:event="store.selected"
|
||||
@close="store.closeDetail()"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, reactive, ref, watch } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
|
||||
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||
import DownloadEventRow from '../downloads/DownloadEventRow.vue'
|
||||
import DownloadDetailModal from '../downloads/DownloadDetailModal.vue'
|
||||
import DownloadStatChips from './DownloadStatChips.vue'
|
||||
import MaintenanceMenu from './MaintenanceMenu.vue'
|
||||
import DownloadsFilterPopover from './DownloadsFilterPopover.vue'
|
||||
|
||||
const route = useRoute()
|
||||
const store = useDownloadsStore()
|
||||
const filterModel = ref({ ...store.filter })
|
||||
|
||||
// Each group's collapsed state persists across refreshes for the
|
||||
// lifetime of the SubscriptionsView (operator-friendly default: all
|
||||
// expanded; collapse what you don't care about right now).
|
||||
const collapsed = reactive({
|
||||
today: false, yesterday: false, week: false, earlier: false,
|
||||
})
|
||||
function toggle(key) {
|
||||
collapsed[key] = !collapsed[key]
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
await Promise.all([
|
||||
store.loadFirst(),
|
||||
store.loadStats(24),
|
||||
])
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
if (route.query.source_id) {
|
||||
filterModel.value = { ...filterModel.value, source_id: Number(route.query.source_id) }
|
||||
}
|
||||
refresh()
|
||||
})
|
||||
|
||||
// Client-side date filter on the loaded page (avoids a backend round-trip
|
||||
// for the date pickers; the existing /api/downloads endpoint can grow
|
||||
// these as proper query params later if a UX need shows up).
|
||||
const filteredEvents = computed(() => {
|
||||
let arr = store.events
|
||||
const from = filterModel.value.from_date
|
||||
const to = filterModel.value.to_date
|
||||
if (from) {
|
||||
const fromTs = new Date(from).getTime()
|
||||
arr = arr.filter((e) => new Date(e.started_at).getTime() >= fromTs)
|
||||
}
|
||||
if (to) {
|
||||
const toTs = new Date(to).getTime() + 24 * 3600 * 1000 - 1
|
||||
arr = arr.filter((e) => new Date(e.started_at).getTime() <= toTs)
|
||||
}
|
||||
return arr
|
||||
})
|
||||
|
||||
// Group events by relative date bucket and pin failed runs to the
|
||||
// top of each bucket. Buckets boundaries are computed against the
|
||||
// operator's local-time start-of-day so "Today" matches their
|
||||
// intuition regardless of the event's stored UTC timestamp.
|
||||
const groups = computed(() => {
|
||||
const now = new Date()
|
||||
const startOfToday = new Date(
|
||||
now.getFullYear(), now.getMonth(), now.getDate(),
|
||||
).getTime()
|
||||
const startOfYesterday = startOfToday - 24 * 3600 * 1000
|
||||
const startOfWeek = startOfToday - 7 * 24 * 3600 * 1000
|
||||
|
||||
const buckets = { today: [], yesterday: [], week: [], earlier: [] }
|
||||
for (const e of filteredEvents.value) {
|
||||
const t = new Date(e.started_at).getTime()
|
||||
if (t >= startOfToday) buckets.today.push(e)
|
||||
else if (t >= startOfYesterday) buckets.yesterday.push(e)
|
||||
else if (t >= startOfWeek) buckets.week.push(e)
|
||||
else buckets.earlier.push(e)
|
||||
}
|
||||
|
||||
function withFailedPinned(items) {
|
||||
const fail = items.filter((e) => e.status === 'error')
|
||||
const rest = items.filter((e) => e.status !== 'error')
|
||||
return [...fail, ...rest]
|
||||
}
|
||||
|
||||
const meta = [
|
||||
{ key: 'today', label: 'Today' },
|
||||
{ key: 'yesterday', label: 'Yesterday' },
|
||||
{ key: 'week', label: 'Last 7 days' },
|
||||
{ key: 'earlier', label: 'Earlier' },
|
||||
]
|
||||
return meta
|
||||
.map(({ key, label }) => {
|
||||
const items = withFailedPinned(buckets[key])
|
||||
const failedCount = items.filter((e) => e.status === 'error').length
|
||||
return { key, label, items, failedCount }
|
||||
})
|
||||
.filter((g) => g.items.length > 0)
|
||||
})
|
||||
|
||||
watch(filterModel, async (m) => {
|
||||
await store.applyFilter({
|
||||
status: m.status,
|
||||
source_id: m.source_id || null,
|
||||
from_date: m.from_date,
|
||||
to_date: m.to_date,
|
||||
})
|
||||
await store.loadStats(24)
|
||||
}, { deep: true })
|
||||
|
||||
async function openDetail(id) {
|
||||
await store.loadOne(id)
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dl__top {
|
||||
display: flex; gap: 8px; align-items: center;
|
||||
margin-bottom: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.fc-dl__filter { margin-bottom: 12px; }
|
||||
.fc-dl__loading, .fc-dl__empty {
|
||||
display: flex; justify-content: center; padding: 3rem 0;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-dl__sentinel {
|
||||
display: flex; justify-content: center; padding: 1rem 0;
|
||||
}
|
||||
|
||||
.fc-dl__group { margin-bottom: 12px; }
|
||||
.fc-dl__group-head {
|
||||
display: flex; align-items: center; gap: 8px;
|
||||
padding: 6px 8px;
|
||||
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
.fc-dl__group-head:hover {
|
||||
background: rgb(var(--v-theme-on-surface) / 0.08);
|
||||
}
|
||||
.fc-dl__group-chev {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-dl__group-label {
|
||||
font-weight: 600;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
flex: 1;
|
||||
}
|
||||
.fc-dl__group-counts {
|
||||
display: flex; gap: 6px; align-items: center;
|
||||
}
|
||||
.fc-dl__group-body { margin-top: 4px; }
|
||||
</style>
|
||||
@@ -0,0 +1,62 @@
|
||||
<template>
|
||||
<v-menu>
|
||||
<template #activator="{ props }">
|
||||
<v-btn v-bind="props" variant="outlined" prepend-icon="mdi-wrench" append-icon="mdi-chevron-down">
|
||||
Maintenance
|
||||
</v-btn>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item
|
||||
prepend-icon="mdi-refresh"
|
||||
title="Retry failed"
|
||||
subtitle="Re-enqueue every failed import task"
|
||||
@click="onRetry"
|
||||
/>
|
||||
<v-list-item
|
||||
prepend-icon="mdi-broom"
|
||||
title="Clear stuck"
|
||||
subtitle="Mark long-running import tasks failed and finalize their batch"
|
||||
@click="onClear"
|
||||
/>
|
||||
<v-list-item
|
||||
:disabled="true"
|
||||
prepend-icon="mdi-download-box"
|
||||
title="Export failed logs"
|
||||
subtitle="CSV dump — v2"
|
||||
>
|
||||
<v-tooltip activator="parent" location="start">Deferred to a future release</v-tooltip>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const emit = defineEmits(['refresh'])
|
||||
const importStore = useImportStore()
|
||||
|
||||
async function onRetry() {
|
||||
try {
|
||||
await importStore.retryFailed()
|
||||
globalThis.window?.__fcToast?.({ text: 'Retry queued', type: 'success' })
|
||||
emit('refresh')
|
||||
} catch (e) {
|
||||
globalThis.window?.__fcToast?.({ text: `Retry failed: ${e?.detail || e?.message || e}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
|
||||
async function onClear() {
|
||||
try {
|
||||
const body = await importStore.clearStuck()
|
||||
const n = body?.cleared ?? 0
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: n ? `Cleared ${n} stuck task${n === 1 ? '' : 's'}` : 'Nothing stuck',
|
||||
type: 'success',
|
||||
})
|
||||
emit('refresh')
|
||||
} catch (e) {
|
||||
globalThis.window?.__fcToast?.({ text: `Clear failed: ${e?.detail || e?.message || e}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -0,0 +1,25 @@
|
||||
<template>
|
||||
<v-chip
|
||||
:color="color"
|
||||
:size="size"
|
||||
:variant="variant"
|
||||
:prepend-icon="icon"
|
||||
>
|
||||
{{ label }}
|
||||
</v-chip>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { platformColor, platformIcon, platformLabel } from '../../utils/platformColor.js'
|
||||
|
||||
const props = defineProps({
|
||||
platform: { type: String, required: true },
|
||||
size: { type: String, default: 'small' },
|
||||
variant: { type: String, default: 'tonal' },
|
||||
})
|
||||
|
||||
const color = computed(() => platformColor(props.platform))
|
||||
const icon = computed(() => platformIcon(props.platform))
|
||||
const label = computed(() => platformLabel(props.platform))
|
||||
</script>
|
||||
@@ -0,0 +1,196 @@
|
||||
<template>
|
||||
<div>
|
||||
<ExtensionKeyBar class="mb-4" />
|
||||
|
||||
<v-alert v-if="credentialsStore.error" type="error" variant="tonal" closable class="mb-4">
|
||||
{{ String(credentialsStore.error) }}
|
||||
</v-alert>
|
||||
|
||||
<h3 class="text-h6 mb-3">Platform credentials</h3>
|
||||
<v-row>
|
||||
<v-col
|
||||
v-for="p in platformsStore.list"
|
||||
:key="p.key"
|
||||
cols="12" md="6"
|
||||
>
|
||||
<CredentialCard
|
||||
:platform="p"
|
||||
:credential="credentialsStore.byPlatform.get(p.key) || null"
|
||||
@replace="openUpload"
|
||||
@remove="confirmRemove"
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<h3 class="text-h6 mb-3 mt-6">Downloader</h3>
|
||||
<v-card variant="outlined">
|
||||
<v-card-text v-if="importStore.settings">
|
||||
<v-row>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-text-field
|
||||
v-model.number="dl.download_rate_limit_seconds"
|
||||
label="Rate limit (seconds between requests)"
|
||||
type="number" step="0.5" min="0"
|
||||
density="compact" hide-details
|
||||
@blur="saveDownloader"
|
||||
/>
|
||||
<div class="fc-help">gallery-dl extractor.sleep. Higher = slower but safer.</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="6">
|
||||
<v-switch
|
||||
v-model="dl.download_validate_files"
|
||||
label="Validate downloaded files (magic-byte check)"
|
||||
density="compact" hide-details color="accent"
|
||||
@change="saveDownloader"
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
</v-card-text>
|
||||
<v-card-text v-else>
|
||||
<v-skeleton-loader type="paragraph" />
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<h3 class="text-h6 mb-3 mt-6">Schedule defaults</h3>
|
||||
<v-card variant="outlined">
|
||||
<v-card-text v-if="importStore.settings">
|
||||
<v-row>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field
|
||||
v-model.number="dl.download_schedule_default_seconds"
|
||||
label="Default check interval (seconds)"
|
||||
type="number" :min="60" :max="86400"
|
||||
density="compact" hide-details
|
||||
@blur="saveDownloader"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Used when a source has no per-source or per-artist override.
|
||||
Default 28800 (8 hours).
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field
|
||||
v-model.number="dl.download_event_retention_days"
|
||||
label="Event retention (days)"
|
||||
type="number" :min="1" :max="3650"
|
||||
density="compact" hide-details
|
||||
@blur="saveDownloader"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Completed download events older than this are deleted nightly.
|
||||
Default 90.
|
||||
</div>
|
||||
</v-col>
|
||||
<v-col cols="12" sm="4">
|
||||
<v-text-field
|
||||
v-model.number="dl.download_failure_warning_threshold"
|
||||
label="Failure warning threshold"
|
||||
type="number" :min="1" :max="100"
|
||||
density="compact" hide-details
|
||||
@blur="saveDownloader"
|
||||
/>
|
||||
<div class="fc-help">
|
||||
Source row badge turns red after this many consecutive
|
||||
failures. Sources are never auto-disabled. Default 5.
|
||||
</div>
|
||||
</v-col>
|
||||
</v-row>
|
||||
<v-alert v-if="importStore.settingsError" type="error" variant="tonal" class="mt-2" closable>
|
||||
{{ importStore.settingsError }}
|
||||
</v-alert>
|
||||
</v-card-text>
|
||||
<v-card-text v-else>
|
||||
<v-skeleton-loader type="paragraph" />
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<CredentialUploadDialog
|
||||
v-model="showUpload"
|
||||
:platform="uploadPlatform"
|
||||
@saved="onSaved"
|
||||
/>
|
||||
|
||||
<v-dialog v-model="removeConfirm.open" max-width="420">
|
||||
<v-card>
|
||||
<v-card-title>Delete {{ removeConfirm.platform?.name }} credential?</v-card-title>
|
||||
<v-card-text>
|
||||
The encrypted credential will be removed permanently. You'll need to
|
||||
re-upload to use this platform again.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="removeConfirm.open = false">Cancel</v-btn>
|
||||
<v-btn color="error" variant="flat" @click="doRemove">Delete</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, reactive, ref, watch } from 'vue'
|
||||
import { usePlatformsStore } from '../../stores/platforms.js'
|
||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import ExtensionKeyBar from '../credentials/ExtensionKeyBar.vue'
|
||||
import CredentialUploadDialog from '../credentials/CredentialUploadDialog.vue'
|
||||
import CredentialCard from './CredentialCard.vue'
|
||||
|
||||
const platformsStore = usePlatformsStore()
|
||||
const credentialsStore = useCredentialsStore()
|
||||
const importStore = useImportStore()
|
||||
|
||||
const showUpload = ref(false)
|
||||
const uploadPlatform = ref(null)
|
||||
const removeConfirm = reactive({ open: false, platform: null })
|
||||
|
||||
const dl = reactive({
|
||||
download_rate_limit_seconds: 3.0,
|
||||
download_validate_files: true,
|
||||
download_schedule_default_seconds: 28800,
|
||||
download_event_retention_days: 90,
|
||||
download_failure_warning_threshold: 5,
|
||||
})
|
||||
|
||||
watch(() => importStore.settings, (s) => { if (s) Object.assign(dl, s) }, { immediate: true })
|
||||
|
||||
onMounted(async () => {
|
||||
await Promise.all([
|
||||
platformsStore.loadAll(),
|
||||
credentialsStore.loadAll(),
|
||||
importStore.loadSettings(),
|
||||
])
|
||||
})
|
||||
|
||||
function openUpload(platform) {
|
||||
uploadPlatform.value = platform
|
||||
showUpload.value = true
|
||||
}
|
||||
|
||||
async function onSaved() {
|
||||
await credentialsStore.loadAll()
|
||||
}
|
||||
|
||||
function confirmRemove(platform) {
|
||||
removeConfirm.platform = platform
|
||||
removeConfirm.open = true
|
||||
}
|
||||
|
||||
async function doRemove() {
|
||||
await credentialsStore.remove(removeConfirm.platform.key)
|
||||
removeConfirm.open = false
|
||||
await credentialsStore.loadAll()
|
||||
}
|
||||
|
||||
async function saveDownloader() {
|
||||
await importStore.patchSettings({ ...dl })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-help {
|
||||
font-size: 12px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin-top: 2px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,521 @@
|
||||
<template>
|
||||
<div>
|
||||
<div class="fc-subs__bar">
|
||||
<v-btn color="accent" prepend-icon="mdi-plus" @click="openAddSource(null)">
|
||||
Add subscription
|
||||
</v-btn>
|
||||
<v-btn variant="outlined" prepend-icon="mdi-account-plus" @click="showArtistDialog = true">
|
||||
New artist
|
||||
</v-btn>
|
||||
<v-spacer />
|
||||
<v-select
|
||||
v-model="statusFilter"
|
||||
:items="STATUS_OPTIONS"
|
||||
density="compact" variant="outlined" hide-details
|
||||
style="max-width: 180px"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="search"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
prepend-inner-icon="mdi-magnify"
|
||||
placeholder="Search subscriptions"
|
||||
style="max-width: 320px"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<v-slide-y-transition>
|
||||
<v-card
|
||||
v-if="selected.length"
|
||||
variant="tonal" color="info"
|
||||
class="fc-subs__bulk mb-3"
|
||||
>
|
||||
<v-card-text class="d-flex align-center pa-3 ga-3">
|
||||
<span class="text-body-2">
|
||||
{{ selected.length }} selected
|
||||
</span>
|
||||
<v-spacer />
|
||||
<v-btn size="small" variant="text" prepend-icon="mdi-toggle-switch" @click="bulkSetEnabled(true)">
|
||||
Enable all
|
||||
</v-btn>
|
||||
<v-btn size="small" variant="text" prepend-icon="mdi-toggle-switch-off-outline" @click="bulkSetEnabled(false)">
|
||||
Disable all
|
||||
</v-btn>
|
||||
<v-btn size="small" variant="text" color="error" prepend-icon="mdi-delete" @click="bulkDelete">
|
||||
Delete
|
||||
</v-btn>
|
||||
<v-btn size="small" variant="text" @click="selected = []">Clear</v-btn>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</v-slide-y-transition>
|
||||
|
||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||
{{ String(store.error) }}
|
||||
</v-alert>
|
||||
|
||||
<div v-if="store.loading && groups.length === 0" class="fc-subs__loading">
|
||||
<v-progress-circular indeterminate color="accent" size="36" />
|
||||
</div>
|
||||
|
||||
<div v-else-if="filteredGroups.length === 0" class="fc-subs__empty">
|
||||
<p v-if="groups.length === 0">No subscriptions yet. Add your first artist.</p>
|
||||
<p v-else>No subscriptions match the current filter.</p>
|
||||
</div>
|
||||
|
||||
<v-card v-else class="fc-subs__card" variant="outlined">
|
||||
<v-data-table
|
||||
:headers="headers"
|
||||
:items="filteredGroups"
|
||||
item-value="key"
|
||||
v-model="selected"
|
||||
v-model:expanded="expanded"
|
||||
:items-per-page="50"
|
||||
:items-per-page-options="ITEMS_PER_PAGE_OPTIONS"
|
||||
density="comfortable"
|
||||
hover show-select show-expand
|
||||
@click:row="onRowClick"
|
||||
>
|
||||
<template #item.name="{ item }">
|
||||
<span class="fc-subs__name">{{ item.artist.name }}</span>
|
||||
</template>
|
||||
|
||||
<template #item.platforms="{ item }">
|
||||
<div class="fc-subs__chips">
|
||||
<PlatformChip
|
||||
v-for="p in item.platforms" :key="p"
|
||||
:platform="p" size="x-small"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<template #item.sources_count="{ item }">
|
||||
<v-chip size="x-small" variant="tonal" label>
|
||||
{{ item.sources.length }}
|
||||
</v-chip>
|
||||
</template>
|
||||
|
||||
<template #item.health="{ item }">
|
||||
<SourceHealthDot
|
||||
v-if="item.worstSource"
|
||||
:source="item.worstSource"
|
||||
:warning-threshold="failureThreshold"
|
||||
/>
|
||||
<span v-else class="fc-subs__zero">—</span>
|
||||
</template>
|
||||
|
||||
<template #item.last_activity="{ item }">
|
||||
<span class="fc-subs__when">{{ formatRelative(item.lastActivity) }}</span>
|
||||
</template>
|
||||
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:loading="anyChecking(item.sources)"
|
||||
@click.stop="checkAll(item)"
|
||||
>
|
||||
<v-icon>mdi-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Check all sources</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn icon size="small" variant="text" @click.stop="openAddSource(item.artist)">
|
||||
<v-icon>mdi-plus</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Add source</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:to="`/posts?artist_id=${item.artist.id}`" @click.stop
|
||||
>
|
||||
<v-icon>mdi-rss</v-icon>
|
||||
<v-tooltip activator="parent" location="top">View posts</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:to="`/artist/${item.artist.slug}`" @click.stop
|
||||
>
|
||||
<v-icon>mdi-account</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Open artist page</v-tooltip>
|
||||
</v-btn>
|
||||
</template>
|
||||
|
||||
<template #expanded-row="{ columns, item }">
|
||||
<tr class="fc-subs__sources-row">
|
||||
<td :colspan="columns.length" class="fc-subs__sources-cell">
|
||||
<v-table density="compact" class="fc-subs__sources-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Platform</th>
|
||||
<th>URL</th>
|
||||
<th>Enabled</th>
|
||||
<th>Last check</th>
|
||||
<th>Next check</th>
|
||||
<th>Errors</th>
|
||||
<th class="text-right">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<SourceRow
|
||||
v-for="s in item.sources" :key="s.id" :source="s"
|
||||
:checking="store.checkingIds.has(s.id)"
|
||||
:warning-threshold="failureThreshold"
|
||||
@edit="openEditSource"
|
||||
@remove="removeSource"
|
||||
@toggle="toggleSourceEnabled"
|
||||
@check="onCheck"
|
||||
/>
|
||||
<tr v-if="item.sources.length === 0">
|
||||
<td colspan="8" class="fc-subs__sources-empty">
|
||||
No sources yet. Click + to add one.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
</v-data-table>
|
||||
</v-card>
|
||||
|
||||
<SourceFormDialog
|
||||
v-model="showSourceDialog"
|
||||
:source="editingSource"
|
||||
:initial-artist="editingArtist"
|
||||
@saved="onSourceSaved"
|
||||
/>
|
||||
<ArtistCreateDialog v-model="showArtistDialog" @created="onArtistCreated" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import { usePlatformsStore } from '../../stores/platforms.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import SourceRow from './SourceRow.vue'
|
||||
import SourceHealthDot from './SourceHealthDot.vue'
|
||||
import SourceFormDialog from './SourceFormDialog.vue'
|
||||
import ArtistCreateDialog from './ArtistCreateDialog.vue'
|
||||
import PlatformChip from './PlatformChip.vue'
|
||||
|
||||
const ITEMS_PER_PAGE_OPTIONS = [
|
||||
{ value: 25, title: '25' },
|
||||
{ value: 50, title: '50' },
|
||||
{ value: 100, title: '100' },
|
||||
{ value: -1, title: 'All' },
|
||||
]
|
||||
const STATUS_OPTIONS = [
|
||||
{ title: 'All status', value: 'all' },
|
||||
{ title: 'Enabled', value: 'enabled' },
|
||||
{ title: 'Disabled', value: 'disabled' },
|
||||
{ title: 'Has errors', value: 'errors' },
|
||||
{ title: 'Stale', value: 'stale' },
|
||||
]
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const store = useSourcesStore()
|
||||
const platformsStore = usePlatformsStore()
|
||||
const importStore = useImportStore()
|
||||
|
||||
const search = ref('')
|
||||
const statusFilter = ref('all')
|
||||
const expanded = ref([])
|
||||
const selected = ref([])
|
||||
const showSourceDialog = ref(false)
|
||||
const editingSource = ref(null)
|
||||
const editingArtist = ref(null)
|
||||
const showArtistDialog = ref(false)
|
||||
|
||||
const artistFilter = computed(() => {
|
||||
const raw = route.query.artist_id
|
||||
return raw == null ? null : Number(raw)
|
||||
})
|
||||
|
||||
const failureThreshold = computed(() =>
|
||||
importStore.settings?.download_failure_warning_threshold ?? 5,
|
||||
)
|
||||
|
||||
async function refresh() {
|
||||
await store.loadAll()
|
||||
await platformsStore.loadAll()
|
||||
if (!importStore.settings) await importStore.loadSettings()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
refresh()
|
||||
if (artistFilter.value != null) {
|
||||
expanded.value = [`artist-${artistFilter.value}`]
|
||||
}
|
||||
})
|
||||
watch(() => route.query.artist_id, refresh)
|
||||
|
||||
const headers = [
|
||||
{ title: 'Subscription', key: 'name', sortable: true, align: 'start' },
|
||||
{ title: 'Platforms', key: 'platforms', sortable: false, align: 'start', width: 240 },
|
||||
{ title: 'Sources', key: 'sources_count', sortable: true, align: 'start', width: 90 },
|
||||
{ title: 'Health', key: 'health', sortable: false, align: 'start', width: 80 },
|
||||
{ title: 'Last activity',key: 'last_activity', sortable: true, align: 'start', width: 140 },
|
||||
{ title: 'Actions', key: 'actions', sortable: false, align: 'end', width: 200 },
|
||||
]
|
||||
|
||||
const groups = computed(() => {
|
||||
const all = store.sourcesByArtistGrouped()
|
||||
return all.map((g) => {
|
||||
const worstSource = pickWorstSource(g.sources, failureThreshold.value)
|
||||
const lastActivity = pickLastActivity(g.sources)
|
||||
const platforms = [...new Set(g.sources.map((s) => s.platform).filter(Boolean))]
|
||||
return {
|
||||
key: `artist-${g.artist.id}`,
|
||||
artist: g.artist,
|
||||
sources: g.sources,
|
||||
sources_count: g.sources.length,
|
||||
platforms,
|
||||
worstSource,
|
||||
lastActivity,
|
||||
name: g.artist.name,
|
||||
last_activity: lastActivity ?? '',
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const filteredGroups = computed(() => {
|
||||
let arr = groups.value
|
||||
if (artistFilter.value != null) {
|
||||
arr = arr.filter((g) => g.artist.id === artistFilter.value)
|
||||
}
|
||||
if (statusFilter.value !== 'all') {
|
||||
arr = arr.filter((g) => groupMatchesStatus(g, statusFilter.value))
|
||||
}
|
||||
const q = search.value?.trim().toLowerCase()
|
||||
if (q) {
|
||||
arr = arr.filter(
|
||||
(g) =>
|
||||
g.artist.name.toLowerCase().includes(q) ||
|
||||
g.sources.some(
|
||||
(s) =>
|
||||
(s.url || '').toLowerCase().includes(q) ||
|
||||
(s.platform || '').toLowerCase().includes(q),
|
||||
),
|
||||
)
|
||||
}
|
||||
return arr
|
||||
})
|
||||
|
||||
function groupMatchesStatus(g, status) {
|
||||
if (status === 'enabled') return g.sources.some((s) => s.enabled)
|
||||
if (status === 'disabled') return g.sources.every((s) => !s.enabled)
|
||||
if (status === 'errors') return g.sources.some((s) => (s.consecutive_failures || 0) > 0)
|
||||
if (status === 'stale') return g.sources.some((s) => !s.last_checked_at)
|
||||
return true
|
||||
}
|
||||
|
||||
function pickLastActivity(sources) {
|
||||
let max = null
|
||||
for (const s of sources) {
|
||||
if (s.last_checked_at && (!max || s.last_checked_at > max)) max = s.last_checked_at
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
function pickWorstSource(sources, threshold) {
|
||||
if (!sources || sources.length === 0) return null
|
||||
function level(s) {
|
||||
if (!s.last_checked_at) return 0
|
||||
const f = s.consecutive_failures || 0
|
||||
if (f === 0) return 1
|
||||
if (f < threshold) return 2
|
||||
return 3
|
||||
}
|
||||
return sources.reduce((worst, s) => (level(s) > level(worst) ? s : worst), sources[0])
|
||||
}
|
||||
|
||||
function formatRelative(iso) {
|
||||
if (!iso) return 'Never'
|
||||
const then = new Date(iso).getTime()
|
||||
const diff = (Date.now() - then) / 1000
|
||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||
return `${Math.floor(diff / 86400)}d ago`
|
||||
}
|
||||
|
||||
function onRowClick(_evt, { item }) {
|
||||
const key = item.key
|
||||
const idx = expanded.value.indexOf(key)
|
||||
if (idx === -1) expanded.value = [...expanded.value, key]
|
||||
else expanded.value = expanded.value.filter((k) => k !== key)
|
||||
}
|
||||
|
||||
function openAddSource(artist) {
|
||||
editingSource.value = null
|
||||
editingArtist.value = artist
|
||||
showSourceDialog.value = true
|
||||
}
|
||||
|
||||
function openEditSource(source) {
|
||||
editingSource.value = source
|
||||
editingArtist.value = {
|
||||
id: source.artist_id, name: source.artist_name, slug: source.artist_slug,
|
||||
}
|
||||
showSourceDialog.value = true
|
||||
}
|
||||
|
||||
async function removeSource(source) {
|
||||
await store.remove(source.id, source.artist_id)
|
||||
await refresh()
|
||||
}
|
||||
|
||||
async function toggleSourceEnabled({ source, enabled }) {
|
||||
await store.update(source.id, { enabled }, source.artist_id)
|
||||
await refresh()
|
||||
}
|
||||
|
||||
async function onSourceSaved() {
|
||||
showSourceDialog.value = false
|
||||
await refresh()
|
||||
}
|
||||
|
||||
function onArtistCreated(artist) {
|
||||
showArtistDialog.value = false
|
||||
openAddSource(artist)
|
||||
}
|
||||
|
||||
async function onCheck(source) {
|
||||
try {
|
||||
const body = await store.checkNow(source.id)
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: `Check enqueued (event #${body.download_event_id})`,
|
||||
type: 'success',
|
||||
})
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) {
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: 'Already running — see Downloads',
|
||||
type: 'info',
|
||||
})
|
||||
router.push({ path: '/subscriptions', query: { tab: 'downloads', source_id: source.id } })
|
||||
} else {
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: `Check failed: ${e?.detail || e?.message || e}`,
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAll(group) {
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
for (const s of group.sources) {
|
||||
if (!s.enabled) continue
|
||||
try {
|
||||
await store.checkNow(s.id)
|
||||
ok += 1
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) conflict += 1
|
||||
}
|
||||
}
|
||||
const parts = []
|
||||
if (ok) parts.push(`${ok} queued`)
|
||||
if (conflict) parts.push(`${conflict} already running`)
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: parts.join(', ') || 'Nothing to check (no enabled sources)',
|
||||
type: 'info',
|
||||
})
|
||||
}
|
||||
|
||||
function anyChecking(sources) {
|
||||
return sources.some((s) => store.checkingIds.has(s.id))
|
||||
}
|
||||
|
||||
function resolveSelectedGroups() {
|
||||
return groups.value.filter((g) => selected.value.includes(g.key))
|
||||
}
|
||||
|
||||
async function bulkSetEnabled(enabled) {
|
||||
const groups = resolveSelectedGroups()
|
||||
let changed = 0
|
||||
for (const g of groups) {
|
||||
for (const s of g.sources) {
|
||||
if (s.enabled === enabled) continue
|
||||
try {
|
||||
await store.update(s.id, { enabled }, s.artist_id)
|
||||
changed += 1
|
||||
} catch { /* keep going */ }
|
||||
}
|
||||
}
|
||||
await refresh()
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: `${changed} source${changed === 1 ? '' : 's'} ${enabled ? 'enabled' : 'disabled'}`,
|
||||
type: 'success',
|
||||
})
|
||||
selected.value = []
|
||||
}
|
||||
|
||||
async function bulkDelete() {
|
||||
const groups = resolveSelectedGroups()
|
||||
const total = groups.reduce((n, g) => n + g.sources.length, 0)
|
||||
if (!globalThis.window?.confirm(
|
||||
`Delete ${total} source${total === 1 ? '' : 's'} across ${groups.length} subscription${groups.length === 1 ? '' : 's'}? Artist rows remain.`,
|
||||
)) return
|
||||
let deleted = 0
|
||||
for (const g of groups) {
|
||||
for (const s of g.sources) {
|
||||
try {
|
||||
await store.remove(s.id, s.artist_id)
|
||||
deleted += 1
|
||||
} catch { /* keep going */ }
|
||||
}
|
||||
}
|
||||
await refresh()
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: `${deleted} source${deleted === 1 ? '' : 's'} deleted`,
|
||||
type: 'success',
|
||||
})
|
||||
selected.value = []
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-subs__bar {
|
||||
display: flex; gap: 0.75rem; align-items: center;
|
||||
padding-bottom: 1rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.fc-subs__loading, .fc-subs__empty {
|
||||
display: flex; justify-content: center; padding: 2rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-subs__card {
|
||||
background: rgb(var(--v-theme-surface));
|
||||
}
|
||||
.fc-subs__name { font-weight: 600; }
|
||||
.fc-subs__chips {
|
||||
display: flex; flex-wrap: wrap; gap: 4px;
|
||||
}
|
||||
.fc-subs__when {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
white-space: nowrap;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.fc-subs__zero {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
opacity: 0.6;
|
||||
}
|
||||
.fc-subs__sources-row td {
|
||||
padding: 0 !important;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-subs__sources-cell {
|
||||
padding-left: 2rem !important;
|
||||
border-top: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||
}
|
||||
.fc-subs__sources-table {
|
||||
background: transparent !important;
|
||||
}
|
||||
.fc-subs__sources-empty {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
text-align: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
.fc-subs__bulk { border-radius: 8px; }
|
||||
</style>
|
||||
@@ -7,8 +7,6 @@ import ArtistView from './views/ArtistView.vue'
|
||||
import SeriesManageView from './views/SeriesManageView.vue'
|
||||
import SeriesReaderView from './views/SeriesReaderView.vue'
|
||||
import SubscriptionsView from './views/SubscriptionsView.vue'
|
||||
import CredentialsView from './views/CredentialsView.vue'
|
||||
import DownloadsView from './views/DownloadsView.vue'
|
||||
import PostsView from './views/PostsView.vue'
|
||||
import ArtistsView from './views/ArtistsView.vue'
|
||||
|
||||
@@ -34,10 +32,16 @@ const routes = [
|
||||
{ path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } },
|
||||
|
||||
// FC-3: subscription backbone
|
||||
// /credentials and /downloads were folded into /subscriptions as subtabs
|
||||
// 2026-05-27 (?tab=settings and ?tab=downloads). The hub view owns the
|
||||
// whole download pipeline domain.
|
||||
{ path: '/posts', name: 'posts', component: PostsView, meta: { title: 'Posts' } },
|
||||
{ path: '/subscriptions', name: 'subscriptions', component: SubscriptionsView, meta: { title: 'Subscriptions' } },
|
||||
{ path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } },
|
||||
{ path: '/downloads', name: 'downloads', component: DownloadsView, meta: { title: 'Downloads' } }
|
||||
|
||||
// Bookmark/back-button safety net for the routes that got folded in
|
||||
// (no meta.title — stay out of TopNav).
|
||||
{ path: '/credentials', redirect: '/subscriptions?tab=settings' },
|
||||
{ path: '/downloads', redirect: '/subscriptions?tab=downloads' }
|
||||
]
|
||||
|
||||
// Browser uses HTML5 history; non-browser (Vitest/SSR) falls back to memory
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref, computed } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
import { usePostsStore } from './posts.js'
|
||||
|
||||
const PAGE = 60
|
||||
|
||||
@@ -15,7 +16,11 @@ export const useArtistStore = defineStore('artist', () => {
|
||||
const notFound = ref(false)
|
||||
let started = false
|
||||
|
||||
async function load(slug) {
|
||||
async function load (slug) {
|
||||
// Cross-artist reset: clear this store AND the posts store so the new
|
||||
// artist doesn't briefly render with the previous artist's content
|
||||
// when the user is on the Posts tab. (Gallery tab uses this artist
|
||||
// store's own images list — cleared above.)
|
||||
overview.value = null
|
||||
images.value = []
|
||||
nextCursor.value = null
|
||||
@@ -23,6 +28,7 @@ export const useArtistStore = defineStore('artist', () => {
|
||||
started = false
|
||||
error.value = null
|
||||
loading.value = true
|
||||
usePostsStore().$reset?.()
|
||||
try {
|
||||
overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`)
|
||||
await loadMoreImages(slug)
|
||||
@@ -34,7 +40,7 @@ export const useArtistStore = defineStore('artist', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMoreImages(slug) {
|
||||
async function loadMoreImages (slug) {
|
||||
if (imagesLoading.value) return
|
||||
if (started && nextCursor.value === null) return
|
||||
imagesLoading.value = true
|
||||
@@ -55,9 +61,13 @@ export const useArtistStore = defineStore('artist', () => {
|
||||
}
|
||||
|
||||
const hasMoreImages = computed(() => !started || nextCursor.value !== null)
|
||||
const postCount = computed(() => overview.value?.post_count ?? null)
|
||||
const imageCount = computed(() => overview.value?.image_count ?? null)
|
||||
const lastAdded = computed(() => overview.value?.date_range?.max ?? null)
|
||||
|
||||
return {
|
||||
overview, images, loading, imagesLoading, error, notFound,
|
||||
hasMoreImages, load, loadMoreImages
|
||||
hasMoreImages, postCount, imageCount, lastAdded,
|
||||
load, loadMoreImages,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
export const useCleanupStore = defineStore('cleanup', () => {
|
||||
const api = useApi()
|
||||
|
||||
// Defaults sourced from ImportSettings on mount. Cards pre-fill from
|
||||
// these so the common case ("apply current import filters
|
||||
// retroactively") is one click; operator can override per-audit.
|
||||
const defaults = ref({
|
||||
min_width: 0,
|
||||
min_height: 0,
|
||||
transparency_threshold: 0.9,
|
||||
single_color_threshold: 0.95,
|
||||
single_color_tolerance: 30,
|
||||
})
|
||||
|
||||
const recentRuns = ref([])
|
||||
|
||||
async function loadDefaults() {
|
||||
const s = await api.get('/api/settings/import')
|
||||
defaults.value = {
|
||||
min_width: s.min_width ?? 0,
|
||||
min_height: s.min_height ?? 0,
|
||||
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||
single_color_threshold: s.single_color_threshold ?? 0.95,
|
||||
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||
}
|
||||
}
|
||||
|
||||
async function previewMinDim(min_width, min_height) {
|
||||
return await api.post('/api/cleanup/min-dimension/preview', {
|
||||
body: { min_width, min_height },
|
||||
})
|
||||
}
|
||||
|
||||
async function deleteMinDim(min_width, min_height, confirm) {
|
||||
return await api.post('/api/cleanup/min-dimension/delete', {
|
||||
body: { min_width, min_height, confirm },
|
||||
})
|
||||
}
|
||||
|
||||
async function startAudit(rule, params) {
|
||||
return await api.post('/api/cleanup/audit', { body: { rule, params } })
|
||||
}
|
||||
|
||||
async function getAudit(id) {
|
||||
return await api.get(`/api/cleanup/audit/${id}`)
|
||||
}
|
||||
|
||||
async function loadHistory(limit = 20) {
|
||||
const body = await api.get(`/api/cleanup/audit?limit=${limit}`)
|
||||
recentRuns.value = body.runs
|
||||
return body.runs
|
||||
}
|
||||
|
||||
async function applyAudit(id, confirm) {
|
||||
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
|
||||
}
|
||||
|
||||
async function cancelAudit(id) {
|
||||
return await api.post(`/api/cleanup/audit/${id}/cancel`)
|
||||
}
|
||||
|
||||
return {
|
||||
defaults, recentRuns,
|
||||
loadDefaults,
|
||||
previewMinDim, deleteMinDim,
|
||||
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
|
||||
}
|
||||
})
|
||||
@@ -8,10 +8,14 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
const events = ref([])
|
||||
const cursor = ref(null)
|
||||
const hasMore = ref(true)
|
||||
const filter = ref({ status: null, source_id: null, artist_id: null })
|
||||
const filter = ref({
|
||||
status: null, source_id: null, artist_id: null,
|
||||
from_date: null, to_date: null,
|
||||
})
|
||||
const selected = ref(null)
|
||||
const loading = ref(false)
|
||||
const error = ref(null)
|
||||
const stats = ref({ pending: 0, running: 0, ok: 0, error: 0, skipped: 0 })
|
||||
|
||||
function _params(extra = {}) {
|
||||
const out = { limit: 50, ...extra }
|
||||
@@ -65,8 +69,13 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
selected.value = null
|
||||
}
|
||||
|
||||
async function loadStats(windowHours = 24) {
|
||||
stats.value = await api.get('/api/downloads/stats', { params: { window_hours: windowHours } })
|
||||
return stats.value
|
||||
}
|
||||
|
||||
return {
|
||||
events, cursor, hasMore, filter, selected, loading, error,
|
||||
loadFirst, loadMore, loadOne, applyFilter, closeDetail,
|
||||
events, cursor, hasMore, filter, selected, loading, error, stats,
|
||||
loadFirst, loadMore, loadOne, applyFilter, closeDetail, loadStats,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -52,11 +52,58 @@ export const useImportStore = defineStore('import', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerScan() {
|
||||
async function triggerScan(mode = 'quick') {
|
||||
if (!['quick', 'deep', 'verify'].includes(mode)) {
|
||||
throw new Error(`unsupported scan mode: ${mode}`)
|
||||
}
|
||||
triggerError.value = null
|
||||
try {
|
||||
await api.post('/api/import/trigger', { body: { mode: 'quick' } })
|
||||
await api.post('/api/import/trigger', { body: { mode } })
|
||||
// Acknowledge immediately so the click isn't invisible. scan_directory
|
||||
// can finalize the batch synchronously when every file in /import is
|
||||
// already on a non-failed ImportTask (operator-flagged 2026-05-25:
|
||||
// 233k existing tasks → all paths in skip-set → files_seen=0 →
|
||||
// batch flashes 'running' for <100ms then 'complete' before the
|
||||
// first refreshStatus() lands; UI never sees the active state).
|
||||
const label = mode === 'deep'
|
||||
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
|
||||
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
|
||||
window.__fcToast?.({ text: label, type: 'success' })
|
||||
await refreshStatus()
|
||||
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
|
||||
// Operator-flagged 2026-05-25: the prior "no new files" message was
|
||||
// misleading because deep scan IS doing work (refresh) even when
|
||||
// there are no new files to import. Surface the real workload count
|
||||
// (imported + refreshed + queued) instead. For quick scan + zero
|
||||
// queued work, fall back to "up to date" instead of the old
|
||||
// implementation-detail-leaking message.
|
||||
setTimeout(async () => {
|
||||
await refreshStatus()
|
||||
await loadTasks(true)
|
||||
if (activeBatch.value || mode === 'verify') return
|
||||
// Batch finalized quickly; figure out what actually happened.
|
||||
// The task list was just refreshed; the freshest row(s) carry
|
||||
// the batch outcome.
|
||||
const batchId = tasks.value[0]?.batch_id
|
||||
const sameBatch = batchId
|
||||
? tasks.value.filter(t => t.batch_id === batchId)
|
||||
: []
|
||||
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
|
||||
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
|
||||
if (mode === 'deep' && sameBatch.length > 0) {
|
||||
window.__fcToast?.({
|
||||
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
|
||||
type: 'info',
|
||||
})
|
||||
} else if (mode === 'quick' && sameBatch.length > 0) {
|
||||
window.__fcToast?.({
|
||||
text: `Quick scan finished — ${newImported} new file(s) queued`,
|
||||
type: 'info',
|
||||
})
|
||||
} else {
|
||||
window.__fcToast?.({ text: 'Library is up to date', type: 'info' })
|
||||
}
|
||||
}, 2000)
|
||||
} catch (e) {
|
||||
triggerError.value = e.message
|
||||
window.__fcToast?.({ text: `Scan failed: ${e.message}`, type: 'error' })
|
||||
@@ -99,6 +146,15 @@ export const useImportStore = defineStore('import', () => {
|
||||
return body
|
||||
}
|
||||
|
||||
// Layer-2 one-shot re-download for a failed task's (corrupt) file.
|
||||
// Returns the endpoint's status dict (refetch_queued / no_source /
|
||||
// already_refetched). Caller surfaces it as a toast.
|
||||
async function refetchTask(taskId) {
|
||||
const body = await api.post(`/api/import/tasks/${taskId}/refetch`)
|
||||
await loadTasks(true)
|
||||
return body
|
||||
}
|
||||
|
||||
const hasMore = computed(() => tasksNextCursor.value !== null)
|
||||
|
||||
return {
|
||||
@@ -108,6 +164,7 @@ export const useImportStore = defineStore('import', () => {
|
||||
triggerError,
|
||||
loadSettings, patchSettings,
|
||||
refreshStatus, triggerScan,
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck,
|
||||
refetchTask,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -6,14 +6,31 @@ export const useModalStore = defineStore('modal', () => {
|
||||
const api = useApi()
|
||||
|
||||
const currentImageId = ref(null)
|
||||
const current = ref(null) // full image detail from API
|
||||
const current = ref(null)
|
||||
const loading = ref(false)
|
||||
const error = ref(null)
|
||||
|
||||
async function open(id) {
|
||||
// Post-scoped cycle. When set, prev/next cycles within this array
|
||||
// (used by PostCard's expanded-mosaic PostImageGrid clicks). When
|
||||
// null, prev/next falls back to current.value.neighbors (the
|
||||
// gallery-store-driven /api/gallery/image/<id> neighbors).
|
||||
const postImageIds = ref(null)
|
||||
const postImageIndex = ref(0)
|
||||
|
||||
async function open (id, opts = {}) {
|
||||
currentImageId.value = id
|
||||
loading.value = true
|
||||
error.value = null
|
||||
// Update post-scoped state if caller passed it; otherwise clear so
|
||||
// the next open() from gallery context uses neighbors mode.
|
||||
if (opts.postImageIds != null) {
|
||||
postImageIds.value = opts.postImageIds
|
||||
postImageIndex.value = opts.postImageIds.indexOf(id)
|
||||
if (postImageIndex.value < 0) postImageIndex.value = 0
|
||||
} else if (opts.clearPostScope !== false && postImageIds.value != null) {
|
||||
postImageIds.value = null
|
||||
postImageIndex.value = 0
|
||||
}
|
||||
try {
|
||||
current.value = await api.get(`/api/gallery/image/${id}`)
|
||||
} catch (e) {
|
||||
@@ -24,41 +41,58 @@ export const useModalStore = defineStore('modal', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function close() {
|
||||
async function close () {
|
||||
currentImageId.value = null
|
||||
current.value = null
|
||||
error.value = null
|
||||
postImageIds.value = null
|
||||
postImageIndex.value = 0
|
||||
}
|
||||
|
||||
async function goPrev() {
|
||||
if (current.value && current.value.neighbors.prev_id) {
|
||||
async function goPrev () {
|
||||
if (postImageIds.value != null) {
|
||||
if (postImageIndex.value > 0) {
|
||||
const newIdx = postImageIndex.value - 1
|
||||
const newId = postImageIds.value[newIdx]
|
||||
postImageIndex.value = newIdx
|
||||
await open(newId, { postImageIds: postImageIds.value })
|
||||
}
|
||||
return
|
||||
}
|
||||
if (current.value && current.value.neighbors?.prev_id) {
|
||||
await open(current.value.neighbors.prev_id)
|
||||
}
|
||||
}
|
||||
async function goNext() {
|
||||
if (current.value && current.value.neighbors.next_id) {
|
||||
|
||||
async function goNext () {
|
||||
if (postImageIds.value != null) {
|
||||
if (postImageIndex.value < postImageIds.value.length - 1) {
|
||||
const newIdx = postImageIndex.value + 1
|
||||
const newId = postImageIds.value[newIdx]
|
||||
postImageIndex.value = newIdx
|
||||
await open(newId, { postImageIds: postImageIds.value })
|
||||
}
|
||||
return
|
||||
}
|
||||
if (current.value && current.value.neighbors?.next_id) {
|
||||
await open(current.value.neighbors.next_id)
|
||||
}
|
||||
}
|
||||
|
||||
async function reloadTags() {
|
||||
async function reloadTags () {
|
||||
if (!currentImageId.value) return
|
||||
const tags = await api.get(`/api/images/${currentImageId.value}/tags`)
|
||||
if (current.value) current.value.tags = tags
|
||||
}
|
||||
|
||||
async function removeTag(tagId) {
|
||||
async function removeTag (tagId) {
|
||||
if (!currentImageId.value) return
|
||||
// Optimistic UI: remove locally first, restore on error.
|
||||
const prev = current.value.tags
|
||||
current.value.tags = current.value.tags.filter(t => t.id !== tagId)
|
||||
try {
|
||||
// FC-2b: removal also records a per-image rejection (suggestions/dismiss
|
||||
// is the rejection-recording endpoint) so the allowlist maintenance
|
||||
// task won't re-apply this tag to this image.
|
||||
await api.delete(`/api/images/${currentImageId.value}/tags/${tagId}`)
|
||||
await api.post(`/api/images/${currentImageId.value}/suggestions/dismiss`, {
|
||||
body: { tag_id: tagId }
|
||||
body: { tag_id: tagId },
|
||||
})
|
||||
} catch (e) {
|
||||
current.value.tags = prev
|
||||
@@ -67,25 +101,36 @@ export const useModalStore = defineStore('modal', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function addExistingTag(tagId) {
|
||||
async function addExistingTag (tagId) {
|
||||
if (!currentImageId.value) return
|
||||
await api.post(`/api/images/${currentImageId.value}/tags`, {
|
||||
body: { tag_id: tagId, source: 'manual' }
|
||||
body: { tag_id: tagId, source: 'manual' },
|
||||
})
|
||||
await reloadTags()
|
||||
}
|
||||
|
||||
async function createAndAdd({ name, kind, fandom_id = null }) {
|
||||
async function createAndAdd ({ name, kind, fandom_id = null }) {
|
||||
const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } })
|
||||
await addExistingTag(tag.id)
|
||||
}
|
||||
|
||||
const isOpen = computed(() => currentImageId.value !== null)
|
||||
const canPrev = computed(() => current.value?.neighbors?.prev_id != null)
|
||||
const canNext = computed(() => current.value?.neighbors?.next_id != null)
|
||||
const canPrev = computed(() => {
|
||||
if (postImageIds.value != null) return postImageIndex.value > 0
|
||||
return current.value?.neighbors?.prev_id != null
|
||||
})
|
||||
const canNext = computed(() => {
|
||||
if (postImageIds.value != null) {
|
||||
return postImageIndex.value < (postImageIds.value.length - 1)
|
||||
}
|
||||
return current.value?.neighbors?.next_id != null
|
||||
})
|
||||
|
||||
return {
|
||||
currentImageId, current, loading, error, isOpen, canPrev, canNext,
|
||||
open, close, goPrev, goNext, reloadTags, removeTag, addExistingTag, createAndAdd
|
||||
currentImageId, current, loading, error,
|
||||
postImageIds, postImageIndex,
|
||||
isOpen, canPrev, canNext,
|
||||
open, close, goPrev, goNext,
|
||||
reloadTags, removeTag, addExistingTag, createAndAdd,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,24 +2,22 @@ import { defineStore } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
// `artist` retired in FC-2d-vii-c (provenance is its own axis), `meta` +
|
||||
// `rating` retired by operator 2026-05-26 (alembic 0023 drops them from
|
||||
// the enum). KIND_COLOR keeps `archive` + `post` so any legacy
|
||||
// system-managed tag still renders with a neutral color.
|
||||
const KIND_OPTIONS = [
|
||||
{ value: 'general', label: 'General', icon: 'mdi-tag' },
|
||||
{ value: 'artist', label: 'Artist', icon: 'mdi-palette' },
|
||||
{ value: 'character', label: 'Character', icon: 'mdi-account-circle' },
|
||||
{ value: 'fandom', label: 'Fandom', icon: 'mdi-book-open-page-variant' },
|
||||
{ value: 'series', label: 'Series', icon: 'mdi-bookshelf' },
|
||||
{ value: 'meta', label: 'Meta', icon: 'mdi-cog-outline' },
|
||||
{ value: 'rating', label: 'Rating', icon: 'mdi-shield-check-outline' }
|
||||
{ value: 'series', label: 'Series', icon: 'mdi-bookshelf' }
|
||||
]
|
||||
|
||||
const KIND_COLOR = {
|
||||
artist: 'accent',
|
||||
character: 'info',
|
||||
fandom: 'secondary',
|
||||
series: 'warning',
|
||||
general: 'on-surface',
|
||||
meta: 'on-surface',
|
||||
rating: 'on-surface',
|
||||
archive: 'on-surface',
|
||||
post: 'on-surface'
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user