Compare commits
170 Commits
v26.05.24.2
...
ext-1.0.4
| Author | SHA1 | Date | |
|---|---|---|---|
| 37e8b796a1 | |||
| 8675f105ad | |||
| 74dac6b960 | |||
| 9e19c081b0 | |||
| 3838f04c16 | |||
| 42b1340324 | |||
| 3b1e2f1ceb | |||
| 8cdf0af0e1 | |||
| ccee344099 | |||
| 0316f92e8b | |||
| 6df74683b3 | |||
| 243e536225 | |||
| 2f16699971 | |||
| a5cb684d34 | |||
| 965a953b2e | |||
| 90c176b195 | |||
| b8d89b9f2a | |||
| 07344e0843 | |||
| 42c33e44f9 | |||
| 4e82208926 | |||
| 85b640f32e | |||
| c7001f4aed | |||
| f827612930 | |||
| 3f0153cba5 | |||
| 52fff00353 | |||
| f3e8f30a8f | |||
| eee107766e | |||
| c14338cbce | |||
| 7a64730bd2 | |||
| 1803a09306 | |||
| 8c36dd28b0 | |||
| 0f7cd3cb76 | |||
| 7b0dd4182c | |||
| 88cfb3dd02 | |||
| fb41b90110 | |||
| 7d84990f6d | |||
| ca55d92c68 | |||
| cce014be3a | |||
| c07effb593 | |||
| a36f72b383 | |||
| 2e8d7c960c | |||
| 992f38ec20 | |||
| 0bc5767a2b | |||
| 397021dcbd | |||
| b0bfbc585a | |||
| 110c1c0e51 | |||
| 6de84d0d60 | |||
| 4e1f208a9f | |||
| 06913eba8e | |||
| b7f693b15e | |||
| ecd0199799 | |||
| 983da9e5b1 | |||
| a41eddae3f | |||
| 7aa7f5a3d6 | |||
| 5d4f223b71 | |||
| 2505b197ae | |||
| 0d0b236ac3 | |||
| a06ada4c9b | |||
| ebd985990c | |||
| 4da8d1d774 | |||
| 05090c6e85 | |||
| 2d4bfa4375 | |||
| 6ed2021ad6 | |||
| 4f2ceaaf31 | |||
| 8a5b337a53 | |||
| 900d878d27 | |||
| fd80d40a34 | |||
| 929d3fc092 | |||
| c0c9e56fb9 | |||
| 3a577d5ade | |||
| 06a2f60c08 | |||
| 0978fbac66 | |||
| f4fe02e346 | |||
| efb142239d | |||
| e766197d99 | |||
| 5587a76606 | |||
| 3872e1dda9 | |||
| 17e19081a2 | |||
| 9814f3dbaf | |||
| 770bcf3aa6 | |||
| 52d7905c43 | |||
| e6ededbe8e | |||
| c06cbc0abe | |||
| b214460fdb | |||
| ac39509a74 | |||
| 3531f373ee | |||
| 36cc0622cb | |||
| e50f92d900 | |||
| ba8d9b112d | |||
| c451061ca5 | |||
| ac55d0e8d8 | |||
| 47d760550d | |||
| 89a89e0ded | |||
| dc3bce7fc1 | |||
| f657582f30 | |||
| 111b952535 | |||
| 4e9aac2c05 | |||
| a0470b5f60 | |||
| b0bb7ae6cc | |||
| 1bbe478fd0 | |||
| 5666fd5ca5 | |||
| 2879ac6f2b | |||
| 3a359f6c5e | |||
| b6a917ac81 | |||
| c361032554 | |||
| 9f54efdedf | |||
| 6b1bb87647 | |||
| 68cffce322 | |||
| 52445eb501 | |||
| 3b3e7565fb | |||
| 9d5abb09f6 | |||
| b8dce6c483 | |||
| 832345a245 | |||
| a0136fa30d | |||
| de1a4b64b7 | |||
| 3f500e592e | |||
| d97e3f9b59 | |||
| 035c49f675 | |||
| e41ab1cca5 | |||
| 42c6b642c2 | |||
| ad3d34a1fc | |||
| b5289ed372 | |||
| f6aa805725 | |||
| f096c9a5fb | |||
| 676a86b514 | |||
| 44cc625d4a | |||
| f7ee122243 | |||
| 7c6f11964a | |||
| 94c60c0af2 | |||
| 6df102b83d | |||
| 2ae01d27e3 | |||
| 718cc79905 | |||
| e78a35d333 | |||
| 83bd3b4b2d | |||
| aecedd9fe4 | |||
| 1e34b1b428 | |||
| 102c21feaa | |||
| 57a338f7e6 | |||
| d04983138a | |||
| 9ec6fdb596 | |||
| 86ad9b80e9 | |||
| 2b05f147f4 | |||
| 70e1e010d1 | |||
| d3d4320ed5 | |||
| 7d42cddb11 | |||
| 1f01c4819a | |||
| 06d527cb92 | |||
| e9ea376aed | |||
| 882cb491ba | |||
| 319e7de547 | |||
| e43312a129 | |||
| 8f2732a56f | |||
| c3e855bd9b | |||
| d1c0b82a22 | |||
| 37fcc74954 | |||
| 6a532c1497 | |||
| 368613068a | |||
| ddbb84d8aa | |||
| 9b252948f9 | |||
| 89d0cb2124 | |||
| 36611cbe00 | |||
| 48ef22445a | |||
| e523d0ac94 | |||
| 541e2bfe6a | |||
| 7782672a51 | |||
| d12b51f6b7 | |||
| 0cda46fcdb | |||
| 79fee98db4 | |||
| b1d68929c5 | |||
| 553567738e |
@@ -2,26 +2,232 @@ name: Build images
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [dev, main]
|
# `:dev` builds dropped 2026-05-26 — operator tests from `:latest` after
|
||||||
|
# merge-to-main, not from the dev branch image. Saves one full docker
|
||||||
|
# build per dev push.
|
||||||
|
branches: [main]
|
||||||
|
# Tag-push triggers an immutable per-version image build (e.g.
|
||||||
|
# `:v26.05.26.5`) — gives a real rollback story alongside the floating
|
||||||
|
# `:main` / `:latest`. Layer reuse keeps the registry-storage cost
|
||||||
|
# negligible per tag. Doesn't overlap with the push-to-main build (that
|
||||||
|
# one publishes `:main` + `:latest`; the tag-push build publishes only
|
||||||
|
# `:<tag>`).
|
||||||
|
tags: ['v*']
|
||||||
|
|
||||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||||
# - write:release (for future release-cutting workflows)
|
# - write:release (for ext-<version> release asset cache)
|
||||||
# - write:issue (for future issue-management automation)
|
# - write:issue (for future issue-management automation)
|
||||||
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-web:
|
# Sign-or-fetch-from-cache: signs the extension via AMO if no ext-<version>
|
||||||
|
# Forgejo release exists yet, otherwise downloads the cached signed XPI.
|
||||||
|
# Result is uploaded as an Actions artifact for build-web to consume.
|
||||||
|
#
|
||||||
|
# Why this lives in build.yml (not a separate workflow): the merge-commit's
|
||||||
|
# docker image tagged `:latest` MUST carry the XPI. A separate sign workflow
|
||||||
|
# racing build.yml leaves `:latest` without the XPI for ~5min (until the
|
||||||
|
# commit-back triggers another build). Inline ordering eliminates the race.
|
||||||
|
# Cache strategy: Forgejo Release Assets — picked 2026-05-25 over Generic
|
||||||
|
# Packages (cleaner API surface) and commit-back-to-side-branch (no extra
|
||||||
|
# branch to manage). AMO blocks re-signing the same version (returns 409),
|
||||||
|
# so signing is intentionally one-shot per version bump.
|
||||||
|
sign-extension:
|
||||||
|
if: github.ref == 'refs/heads/main'
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Resolve extension version
|
||||||
|
id: extver
|
||||||
|
run: |
|
||||||
|
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||||
|
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Resolved extension version: $VERSION"
|
||||||
|
|
||||||
|
- name: Check Forgejo release-asset cache
|
||||||
|
id: cache
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
VERSION=${{ steps.extver.outputs.version }}
|
||||||
|
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||||
|
-H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||||
|
echo "Tag lookup HTTP status: $STATUS"
|
||||||
|
# JSON parsing via python (ci-python:3.14 has stdlib json; jq is
|
||||||
|
# not in the image and adding it per ci-requirements.md is not
|
||||||
|
# warranted for a single consumer — operator-flagged 2026-05-26
|
||||||
|
# after a sign job failed with `jq: not found`).
|
||||||
|
if [ "$STATUS" = "200" ]; then
|
||||||
|
ASSET_ID=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['id'] if xpis else '')")
|
||||||
|
if [ -n "$ASSET_ID" ]; then
|
||||||
|
echo "cached=true" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "asset_id=$ASSET_ID" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Cached XPI exists at ext-$VERSION (asset id $ASSET_ID); skipping AMO sign"
|
||||||
|
else
|
||||||
|
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Release ext-$VERSION exists but has no .xpi asset; will re-sign + re-upload"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "cached=false" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "No release named ext-$VERSION; will sign via AMO and upload"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# No "download cached XPI in sign-extension" step: build-web
|
||||||
|
# fetches directly from the Forgejo ext-<version> release asset
|
||||||
|
# (removed 2026-05-26 alongside the actions/upload-artifact
|
||||||
|
# removal — sign-extension's job is just to ensure the cache
|
||||||
|
# exists on Forgejo; the build-web side reads it independently).
|
||||||
|
|
||||||
|
- name: Sign via AMO (cache miss)
|
||||||
|
if: steps.cache.outputs.cached != 'true'
|
||||||
|
run: |
|
||||||
|
cd extension && npm install --no-save --no-audit --no-fund && npm run sign
|
||||||
|
env:
|
||||||
|
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||||
|
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||||
|
|
||||||
|
- name: Upload signed XPI to ext-<version> release (cache miss)
|
||||||
|
if: steps.cache.outputs.cached != 'true'
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
VERSION=${{ steps.extver.outputs.version }}
|
||||||
|
# AMO renames signed XPIs with its internal addon-id-safe-string;
|
||||||
|
# canonicalize to fabledcurator-<version>.xpi so the FC server's
|
||||||
|
# whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
|
||||||
|
# keeps working.
|
||||||
|
SIGNED=$(ls extension/web-ext-artifacts/*.xpi | head -1)
|
||||||
|
XPI="extension/web-ext-artifacts/fabledcurator-$VERSION.xpi"
|
||||||
|
cp "$SIGNED" "$XPI"
|
||||||
|
# Find-or-create the ext-<version> release. Track whether WE
|
||||||
|
# created it so an upload failure below can roll back (don't
|
||||||
|
# leave an empty release tombstone that the next run's
|
||||||
|
# cache-check mistakes for a partial-failure state).
|
||||||
|
STATUS=$(curl -s -o release.json -w "%{http_code}" \
|
||||||
|
-H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
|
||||||
|
if [ "$STATUS" = "200" ]; then
|
||||||
|
CREATED_BY_US=false
|
||||||
|
else
|
||||||
|
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||||
|
-d "{\"tag_name\":\"ext-$VERSION\",\"name\":\"Extension $VERSION (signed XPI cache)\",\"body\":\"Internal cache for the signed XPI consumed by build.yml's build-web job. Not a user-facing FC release.\",\"target_commitish\":\"main\"}" \
|
||||||
|
-o release.json \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases"
|
||||||
|
CREATED_BY_US=true
|
||||||
|
fi
|
||||||
|
RELEASE_ID=$(python3 -c "import json; print(json.load(open('release.json'))['id'])")
|
||||||
|
test -n "$RELEASE_ID"
|
||||||
|
# Rollback-on-failure: if the asset upload fails AND we just
|
||||||
|
# created the release in this run, delete it. Prevents an empty
|
||||||
|
# ext-<version> release from poisoning the next workflow run
|
||||||
|
# (operator-flagged 2026-05-26 — without rollback the next run
|
||||||
|
# saw 'release exists, no asset → cache miss → sign' which AMO
|
||||||
|
# then rejected with 409 'Version already exists').
|
||||||
|
rollback_if_we_created() {
|
||||||
|
if [ "$CREATED_BY_US" = "true" ]; then
|
||||||
|
echo "Rolling back: deleting just-created release $RELEASE_ID"
|
||||||
|
curl -s -X DELETE -H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID" || true
|
||||||
|
curl -s -X DELETE -H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/tags/ext-$VERSION" || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap 'rollback_if_we_created' EXIT
|
||||||
|
HTTP_CODE=$(curl -s -X POST -H "Authorization: token $TOKEN" \
|
||||||
|
-F "attachment=@$XPI" \
|
||||||
|
-o /dev/null -w "%{http_code}" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi")
|
||||||
|
if [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "200" ]; then
|
||||||
|
echo "Asset upload failed with HTTP $HTTP_CODE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Upload succeeded — clear the rollback trap.
|
||||||
|
trap - EXIT
|
||||||
|
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
|
||||||
|
|
||||||
|
# No actions/upload-artifact step: Forgejo Actions (and our
|
||||||
|
# act_runner) doesn't support upload-artifact@v4+ (GHES limitation
|
||||||
|
# surfaced 2026-05-26). Instead build-web reads the signed XPI
|
||||||
|
# straight from the ext-<version> Forgejo release we just uploaded
|
||||||
|
# to. Same source of truth; no double-store.
|
||||||
|
|
||||||
|
build-web:
|
||||||
|
needs: [sign-extension]
|
||||||
|
# sign-extension is main-only; on dev it's skipped, build-web still runs.
|
||||||
|
if: always() && (needs.sign-extension.result == 'success' || needs.sign-extension.result == 'skipped')
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Download signed XPI from Forgejo release asset (main + tags)
|
||||||
|
# Fires on main-push AND on tag-push. Tag-push builds re-package the
|
||||||
|
# same source code as the preceding main-push build but with an
|
||||||
|
# immutable version tag — they need the XPI too, otherwise the
|
||||||
|
# versioned image ships without the signed extension.
|
||||||
|
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||||
|
# Look up the ext-<version> release; extract the .xpi asset's
|
||||||
|
# browser_download_url (Forgejo's /releases/assets/<id> endpoint
|
||||||
|
# returns ASSET METADATA, not the binary blob — operator-flagged
|
||||||
|
# 2026-05-26: my prior code curl'd the metadata endpoint without
|
||||||
|
# -f and wrote the resulting 404-page-not-found text into
|
||||||
|
# fabledcurator-*.xpi, which Firefox then rejected as "corrupt").
|
||||||
|
# browser_download_url is the canonical binary endpoint and is
|
||||||
|
# also publicly accessible (no token needed) but we pass the
|
||||||
|
# token anyway for symmetry with private-repo support.
|
||||||
|
curl -sf -H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \
|
||||||
|
-o release.json
|
||||||
|
DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])")
|
||||||
|
test -n "$DOWNLOAD_URL"
|
||||||
|
echo "Downloading XPI from: $DOWNLOAD_URL"
|
||||||
|
mkdir -p frontend/public/extension
|
||||||
|
DEST="frontend/public/extension/fabledcurator-$VERSION.xpi"
|
||||||
|
# -f = fail on HTTP error (prevents silent corruption like the
|
||||||
|
# 2026-05-26 incident); -L = follow redirects.
|
||||||
|
curl -sfL -H "Authorization: token $TOKEN" -o "$DEST" "$DOWNLOAD_URL"
|
||||||
|
# Sanity check: the binary should start with the ZIP magic (PK\x03\x04).
|
||||||
|
# If it's anything else, the next docker build will ship a corrupt XPI.
|
||||||
|
MAGIC=$(head -c 2 "$DEST" | od -An -c | tr -d ' \n')
|
||||||
|
if [ "$MAGIC" != "PK" ]; then
|
||||||
|
echo "ERROR: downloaded XPI does not start with ZIP magic 'PK' (got '$MAGIC')"
|
||||||
|
echo "File contents preview:"
|
||||||
|
head -c 200 "$DEST"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||||
|
ls -la frontend/public/extension/
|
||||||
|
|
||||||
- name: Determine tag
|
- name: Determine tag
|
||||||
id: tag
|
id: tag
|
||||||
run: |
|
run: |
|
||||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
# Three trigger shapes:
|
||||||
|
# refs/tags/v… → tag-push: publish ONLY the immutable version
|
||||||
|
# tag (e.g. :v26.05.26.5). Don't touch :latest;
|
||||||
|
# that already got published by the main-push
|
||||||
|
# build for the merge commit.
|
||||||
|
# refs/heads/main → push to main (incl. PR merge commits):
|
||||||
|
# publish :main + :latest (floating).
|
||||||
|
# anything else → safety net; shouldn't fire given the `on:`
|
||||||
|
# config above (dev was dropped). Tag :dev to
|
||||||
|
# surface the unexpected run in the registry.
|
||||||
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
||||||
@@ -52,7 +258,13 @@ jobs:
|
|||||||
- name: Determine tag
|
- name: Determine tag
|
||||||
id: tag
|
id: tag
|
||||||
run: |
|
run: |
|
||||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
# Mirrors build-web's three-shape logic (tag-push / main-push /
|
||||||
|
# safety-net dev). The -ml image follows the same release cadence
|
||||||
|
# as the web image.
|
||||||
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
||||||
|
|||||||
+183
-24
@@ -8,8 +8,10 @@ name: CI
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [dev, main]
|
branches: [dev, main]
|
||||||
pull_request:
|
# pull_request trigger intentionally absent — with branches: [dev, main]
|
||||||
branches: [main]
|
# above, every PR commit already fires CI via the push event on dev. Adding
|
||||||
|
# pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
|
||||||
|
# (single-operator Forgejo repo) so push coverage is complete.
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
backend-lint-and-test:
|
backend-lint-and-test:
|
||||||
@@ -24,11 +26,30 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
# Cache step removed 2026-05-26: act_runner's cache backend has been
|
||||||
|
# broken on this homelab runner since 2026-05-15 (first as request-
|
||||||
|
# timeout warnings, then as hard "Cannot find module .../dist/restore/
|
||||||
|
# index.js" failures that tank the whole job). The cache step targeted
|
||||||
|
# ~/.cache/pip but the install below uses `uv pip install` primarily,
|
||||||
|
# whose own cache lives at ~/.cache/uv — so the cache step's real
|
||||||
|
# benefit was marginal even when working. Cost of removal: ~30s of
|
||||||
|
# wheel downloads per job. Future re-enable: mount ~/.cache/uv as a
|
||||||
|
# docker volume at the runner level (skips actions/cache entirely),
|
||||||
|
# or fix the runner-side cache backend (clear /var/run/act/actions/*,
|
||||||
|
# pin act_runner version, etc.).
|
||||||
|
|
||||||
- name: Install Python deps
|
- name: Install Python deps
|
||||||
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
||||||
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
||||||
# versions live on the runner image, not here.
|
# versions live on the runner image, not here.
|
||||||
run: pip install -r requirements.txt pytest pytest-asyncio
|
# uv: 5-10x faster wheel resolve than pip for cold caches.
|
||||||
|
# Falls back to pip install on uv-missing runners (older images).
|
||||||
|
run: |
|
||||||
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||||
|
else
|
||||||
|
pip install -r requirements.txt pytest pytest-asyncio
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Ruff lint
|
- name: Ruff lint
|
||||||
run: ruff check backend/ tests/ alembic/
|
run: ruff check backend/ tests/ alembic/
|
||||||
@@ -57,17 +78,28 @@ jobs:
|
|||||||
- run: npm run test:unit
|
- run: npm run test:unit
|
||||||
- run: npm run build
|
- run: npm run build
|
||||||
|
|
||||||
integration:
|
# Integration suite split into THREE parallel shards (2026-05-25, runner
|
||||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
|
||||||
# default bridge with NO service-name DNS, and publishing fixed host
|
# set and runs alembic + a disjoint subset of integration tests. Shards
|
||||||
# ports collides with the operator's running docker-compose dev stack on
|
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
|
||||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
# stays single-threaded per shard but multiple shards run in parallel
|
||||||
# each service by its bridge IP — discovered at runtime via the mounted
|
# wall-clock. Approximate split — rebalance once --durations=15 output
|
||||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
# reveals which shard is the long pole.
|
||||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
#
|
||||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
# Each shard's docker-ps filter uses its own unique job name to scope
|
||||||
# values don't depend on cross-step env passing. Pattern documented in
|
# service-container resolution. act_runner appears to strip underscores
|
||||||
# FabledRulebook/forgejo.md "CI philosophy".
|
# from job names when building container labels — `int_api` yielded
|
||||||
|
# zero matches on 2026-05-25 — so shards use no-separator names
|
||||||
|
# (`intapi`, `intimp`, `intcore`) instead. Each step prints
|
||||||
|
# `docker ps -a` first so a future naming-convention shift surfaces in
|
||||||
|
# the log without another guess-and-push cycle.
|
||||||
|
#
|
||||||
|
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
|
||||||
|
# done — per ci-requirements.md, FC is the only Python consumer of that
|
||||||
|
# image and the CI-Runner project's "add deps to image when used by >1
|
||||||
|
# project" rule keeps the install per-job.
|
||||||
|
|
||||||
|
intapi:
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
@@ -98,15 +130,14 @@ jobs:
|
|||||||
--health-retries 10
|
--health-retries 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
- name: API integration shard (resolve service IPs, migrate, test)
|
||||||
- name: Integration suite (resolve service IPs, migrate, test)
|
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
# Scope to THIS job's service containers (act_runner names them
|
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||||
# ...JOB-integration...); the operator's compose stack uses the
|
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||||
# same images but different names, so it won't match.
|
echo "=== end landscape ==="
|
||||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||||
test -n "$PG" && test -n "$RD"
|
test -n "$PG" && test -n "$RD"
|
||||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
@@ -114,11 +145,139 @@ jobs:
|
|||||||
export DB_HOST="$PG_IP"
|
export DB_HOST="$PG_IP"
|
||||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
|
||||||
for i in $(seq 1 60); do
|
for i in $(seq 1 60); do
|
||||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
pip install -r requirements.txt pytest pytest-asyncio
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||||
|
else
|
||||||
|
pip install -r requirements.txt pytest pytest-asyncio
|
||||||
|
fi
|
||||||
alembic upgrade head
|
alembic upgrade head
|
||||||
pytest tests/ -v -m integration
|
pytest tests/test_api_*.py -v -m integration --durations=15
|
||||||
|
|
||||||
|
intimp:
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
env:
|
||||||
|
DB_USER: fabledcurator
|
||||||
|
DB_PASSWORD: ci_integration
|
||||||
|
DB_PORT: "5432"
|
||||||
|
DB_NAME: fabledcurator_test
|
||||||
|
SECRET_KEY: ci_integration_placeholder
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: pgvector/pgvector:pg16
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: fabledcurator
|
||||||
|
POSTGRES_PASSWORD: ci_integration
|
||||||
|
POSTGRES_DB: fabledcurator_test
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U fabledcurator"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
options: >-
|
||||||
|
--health-cmd "redis-cli ping"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Importer integration shard (resolve service IPs, migrate, test)
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||||
|
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||||
|
echo "=== end landscape ==="
|
||||||
|
PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||||
|
RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||||
|
test -n "$PG" && test -n "$RD"
|
||||||
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
|
test -n "$PG_IP" && test -n "$RD_IP"
|
||||||
|
export DB_HOST="$PG_IP"
|
||||||
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||||
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||||
|
else
|
||||||
|
pip install -r requirements.txt pytest pytest-asyncio
|
||||||
|
fi
|
||||||
|
alembic upgrade head
|
||||||
|
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
|
||||||
|
|
||||||
|
intcore:
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
env:
|
||||||
|
DB_USER: fabledcurator
|
||||||
|
DB_PASSWORD: ci_integration
|
||||||
|
DB_PORT: "5432"
|
||||||
|
DB_NAME: fabledcurator_test
|
||||||
|
SECRET_KEY: ci_integration_placeholder
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: pgvector/pgvector:pg16
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: fabledcurator
|
||||||
|
POSTGRES_PASSWORD: ci_integration
|
||||||
|
POSTGRES_DB: fabledcurator_test
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U fabledcurator"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
options: >-
|
||||||
|
--health-cmd "redis-cli ping"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||||
|
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||||
|
echo "=== end landscape ==="
|
||||||
|
PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||||
|
RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||||
|
test -n "$PG" && test -n "$RD"
|
||||||
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
|
test -n "$PG_IP" && test -n "$RD_IP"
|
||||||
|
export DB_HOST="$PG_IP"
|
||||||
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||||
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||||
|
else
|
||||||
|
pip install -r requirements.txt pytest pytest-asyncio
|
||||||
|
fi
|
||||||
|
alembic upgrade head
|
||||||
|
pytest tests/ -v -m integration --durations=15 \
|
||||||
|
--ignore-glob='tests/test_api_*.py' \
|
||||||
|
--ignore-glob='tests/test_importer*.py' \
|
||||||
|
--ignore-glob='tests/test_import_*.py' \
|
||||||
|
--ignore-glob='tests/test_migration_*.py' \
|
||||||
|
--ignore-glob='tests/test_phash_*.py' \
|
||||||
|
--ignore-glob='tests/test_sidecar_*.py' \
|
||||||
|
--ignore-glob='tests/test_scan_*.py' \
|
||||||
|
--ignore-glob='tests/test_archive_extractor.py' \
|
||||||
|
--ignore-glob='tests/test_backfill_phash.py'
|
||||||
|
|||||||
@@ -1,8 +1,19 @@
|
|||||||
name: extension
|
name: extension
|
||||||
|
# Lint-only workflow. The sign-and-publish dance moved into build.yml's
|
||||||
|
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
||||||
|
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
||||||
|
# eliminating the prior race between build.yml and a separate extension.yml.
|
||||||
|
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [dev, main]
|
branches: [dev, main]
|
||||||
paths: ['extension/**']
|
paths:
|
||||||
|
- 'extension/**'
|
||||||
|
- '.forgejo/workflows/extension.yml'
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- 'extension/**'
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
@@ -16,44 +27,3 @@ jobs:
|
|||||||
run: cd extension && npm install --no-save --no-audit --no-fund
|
run: cd extension && npm install --no-save --no-audit --no-fund
|
||||||
- name: Lint
|
- name: Lint
|
||||||
run: cd extension && npm run lint
|
run: cd extension && npm run lint
|
||||||
|
|
||||||
sign-and-publish:
|
|
||||||
needs: lint
|
|
||||||
if: github.ref == 'refs/heads/main'
|
|
||||||
runs-on: python-ci
|
|
||||||
container:
|
|
||||||
image: node:22-bookworm-slim
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.RELEASE_TOKEN }}
|
|
||||||
- name: Install web-ext + git
|
|
||||||
run: |
|
|
||||||
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
|
|
||||||
cd extension && npm install --no-save --no-audit --no-fund
|
|
||||||
- name: Sign XPI
|
|
||||||
run: cd extension && npm run sign
|
|
||||||
env:
|
|
||||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
|
||||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
|
||||||
- name: Commit signed XPI to frontend/public/extension/
|
|
||||||
run: |
|
|
||||||
set -e
|
|
||||||
XPI=$(ls extension/web-ext-artifacts/fabledcurator-*.xpi | head -1)
|
|
||||||
if [ -z "$XPI" ]; then
|
|
||||||
echo "No XPI produced by web-ext sign — exiting"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
mkdir -p frontend/public/extension
|
|
||||||
cp "$XPI" frontend/public/extension/
|
|
||||||
# Also copy as -latest.xpi so the FC server can serve a stable URL.
|
|
||||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
|
||||||
git config user.name "FC extension CI"
|
|
||||||
git config user.email "noreply@fabledsword.com"
|
|
||||||
git add frontend/public/extension/
|
|
||||||
if git diff --cached --quiet; then
|
|
||||||
echo "No changes to commit"
|
|
||||||
else
|
|
||||||
git commit -m "ext: publish signed XPI $(basename $XPI)"
|
|
||||||
git push origin HEAD:main
|
|
||||||
fi
|
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""fc3i: task_run table
|
||||||
|
|
||||||
|
Revision ID: 0016
|
||||||
|
Revises: 0015
|
||||||
|
Create Date: 2026-05-24
|
||||||
|
|
||||||
|
Additive only. New table records every Celery task attempt via signal
|
||||||
|
handlers (backend.app.celery_signals). Status is plain String(16) not
|
||||||
|
Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard-
|
||||||
|
fail at INSERT, String columns extend cleanly).
|
||||||
|
|
||||||
|
Composite indexes anticipate the three dashboard panes:
|
||||||
|
- (queue, started_at desc) — per-lane recent activity
|
||||||
|
- (status, started_at desc) — recent failures pane
|
||||||
|
- (task_name, started_at desc) — drill-down by task
|
||||||
|
|
||||||
|
Indexed columns get individual indexes via `index=True` on the model;
|
||||||
|
the composites below cover the multi-column lookups.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0016"
|
||||||
|
down_revision: Union[str, None] = "0015"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"task_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("celery_task_id", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("queue", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("task_name", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("target_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), nullable=False,
|
||||||
|
server_default="running",
|
||||||
|
),
|
||||||
|
sa.Column("error_type", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("retry_count", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("worker_hostname", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("args_summary", sa.String(length=255), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Single-column indexes (matches Mapped[...].index=True on model).
|
||||||
|
op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"])
|
||||||
|
op.create_index("ix_task_run_queue", "task_run", ["queue"])
|
||||||
|
op.create_index("ix_task_run_task_name", "task_run", ["task_name"])
|
||||||
|
op.create_index("ix_task_run_started_at", "task_run", ["started_at"])
|
||||||
|
op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"])
|
||||||
|
op.create_index("ix_task_run_status", "task_run", ["status"])
|
||||||
|
|
||||||
|
# Composite indexes for dashboard query patterns.
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_queue_started",
|
||||||
|
"task_run", ["queue", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_status_started",
|
||||||
|
"task_run", ["status", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_name_started",
|
||||||
|
"task_run", ["task_name", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_task_run_name_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_status_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_queue_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_status", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_finished_at", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_started_at", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_task_name", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_queue", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_celery_task_id", table_name="task_run")
|
||||||
|
op.drop_table("task_run")
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""fc3h: backup_run table
|
||||||
|
|
||||||
|
Revision ID: 0017
|
||||||
|
Revises: 0016
|
||||||
|
Create Date: 2026-05-24
|
||||||
|
|
||||||
|
Additive. New table records every backup/restore attempt with artifact
|
||||||
|
metadata. Lifecycle tracking lives in task_run from FC-3i; this is
|
||||||
|
artifact-only (paths, sizes, tag, restore lineage).
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0017"
|
||||||
|
down_revision: Union[str, None] = "0016"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"backup_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), nullable=False,
|
||||||
|
server_default="pending",
|
||||||
|
),
|
||||||
|
sa.Column("tag", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("triggered_by", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("sql_path", sa.Text(), nullable=True),
|
||||||
|
sa.Column("tar_path", sa.Text(), nullable=True),
|
||||||
|
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"manifest", sa.JSON(), nullable=False, server_default="{}",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"restored_from_id", sa.Integer(),
|
||||||
|
sa.ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Single-column indexes (matches Mapped[...].index=True).
|
||||||
|
op.create_index("ix_backup_run_kind", "backup_run", ["kind"])
|
||||||
|
op.create_index("ix_backup_run_status", "backup_run", ["status"])
|
||||||
|
op.create_index("ix_backup_run_tag", "backup_run", ["tag"])
|
||||||
|
op.create_index("ix_backup_run_started_at", "backup_run", ["started_at"])
|
||||||
|
op.create_index("ix_backup_run_finished_at", "backup_run", ["finished_at"])
|
||||||
|
|
||||||
|
# Composite indexes for dashboard query patterns.
|
||||||
|
op.create_index(
|
||||||
|
"ix_backup_run_kind_started",
|
||||||
|
"backup_run", ["kind", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_backup_run_status_finished",
|
||||||
|
"backup_run", ["status", sa.text("finished_at DESC")],
|
||||||
|
)
|
||||||
|
# Partial index: only tagged rows participate in retention-exempt query.
|
||||||
|
op.create_index(
|
||||||
|
"ix_backup_run_tag_partial",
|
||||||
|
"backup_run", ["tag"],
|
||||||
|
postgresql_where=sa.text("tag IS NOT NULL"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_backup_run_tag_partial", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_status_finished", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_kind_started", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_finished_at", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_started_at", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_tag", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_status", table_name="backup_run")
|
||||||
|
op.drop_index("ix_backup_run_kind", table_name="backup_run")
|
||||||
|
op.drop_table("backup_run")
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""fc3h: backup_* knobs on import_settings
|
||||||
|
|
||||||
|
Revision ID: 0018
|
||||||
|
Revises: 0017
|
||||||
|
Create Date: 2026-05-24
|
||||||
|
|
||||||
|
Adds four columns to the singleton import_settings row:
|
||||||
|
- backup_db_nightly_enabled (default False — opt-in)
|
||||||
|
- backup_db_nightly_hour_utc (default 3)
|
||||||
|
- backup_db_keep_last_n (default 14)
|
||||||
|
- backup_images_keep_last_n (default 3)
|
||||||
|
|
||||||
|
server_default ensures the singleton row is backfilled in place
|
||||||
|
without an UPDATE statement.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0018"
|
||||||
|
down_revision: Union[str, None] = "0017"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"backup_db_nightly_enabled", sa.Boolean(),
|
||||||
|
nullable=False, server_default=sa.false(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"backup_db_nightly_hour_utc", sa.Integer(),
|
||||||
|
nullable=False, server_default="3",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"backup_db_keep_last_n", sa.Integer(),
|
||||||
|
nullable=False, server_default="14",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"backup_images_keep_last_n", sa.Integer(),
|
||||||
|
nullable=False, server_default="3",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_settings", "backup_images_keep_last_n")
|
||||||
|
op.drop_column("import_settings", "backup_db_keep_last_n")
|
||||||
|
op.drop_column("import_settings", "backup_db_nightly_hour_utc")
|
||||||
|
op.drop_column("import_settings", "backup_db_nightly_enabled")
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""import_batch.refreshed counter for deep-scan sidecar re-application
|
||||||
|
|
||||||
|
Revision ID: 0019
|
||||||
|
Revises: 0018
|
||||||
|
Create Date: 2026-05-25
|
||||||
|
|
||||||
|
Adds a `refreshed` counter to `import_batch`, mirroring the existing
|
||||||
|
`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now
|
||||||
|
re-applies sidecar metadata to already-imported files (the IR feature
|
||||||
|
that didn't make the FC port the first time); a "refreshed" outcome
|
||||||
|
increments this counter so the UI can surface "X new, Y refreshed"
|
||||||
|
instead of the misleading "Scan complete — no new files" message.
|
||||||
|
|
||||||
|
server_default=0 backfills existing rows in place — no UPDATE needed.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0019"
|
||||||
|
down_revision: Union[str, None] = "0018"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"import_batch",
|
||||||
|
sa.Column(
|
||||||
|
"refreshed", sa.Integer(),
|
||||||
|
nullable=False, server_default=sa.text("0"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_batch", "refreshed")
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""fc-cleanup: library_audit_run table for async transparency/single_color audits
|
||||||
|
|
||||||
|
Revision ID: 0020
|
||||||
|
Revises: 0019
|
||||||
|
Create Date: 2026-05-26
|
||||||
|
|
||||||
|
The table backs the async audit lifecycle: rule + params snapshot, status
|
||||||
|
state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and
|
||||||
|
the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs
|
||||||
|
per row by the scan task (oversize = rule too aggressive, operator narrows
|
||||||
|
before re-running).
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0020"
|
||||||
|
down_revision: Union[str, None] = "0019"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"library_audit_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("rule", sa.String(32), nullable=False),
|
||||||
|
sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(16),
|
||||||
|
nullable=False, server_default="running",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True),
|
||||||
|
nullable=False, server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"scanned_count", sa.Integer(),
|
||||||
|
nullable=False, server_default="0",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"matched_count", sa.Integer(),
|
||||||
|
nullable=False, server_default="0",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"matched_ids", postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
nullable=False, server_default=sa.text("'[]'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_audit_run_rule", "library_audit_run", ["rule"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_audit_run_status", "library_audit_run", ["status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_library_audit_run_status", table_name="library_audit_run")
|
||||||
|
op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run")
|
||||||
|
op.drop_table("library_audit_run")
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""provenance-race: dedupe + UNIQUE(image_record_id, post_id) on image_provenance
|
||||||
|
|
||||||
|
Revision ID: 0021
|
||||||
|
Revises: 0020
|
||||||
|
Create Date: 2026-05-26
|
||||||
|
|
||||||
|
Closes the race in Importer._apply_sidecar's existence-check + INSERT pattern.
|
||||||
|
Two workers writing for the same (image, post) pair both saw no existing row
|
||||||
|
and both inserted, leaving duplicates that then broke .scalar_one_or_none()
|
||||||
|
on every subsequent deep-scan rederive against those images
|
||||||
|
(MultipleResultsFound). Most plausibly seeded when the 5-min recovery sweep
|
||||||
|
re-enqueued a still-running long-import task and the second worker collided
|
||||||
|
with the first inside _apply_sidecar.
|
||||||
|
|
||||||
|
Migration steps:
|
||||||
|
1. DELETE all but min(id) per (image_record_id, post_id) pair. Operator's
|
||||||
|
DB had 2 affected pairs at write-time; harmless no-op if zero.
|
||||||
|
2. Add UNIQUE constraint so the importer's new savepoint+IntegrityError
|
||||||
|
recovery path can trip on collision and re-select, mirroring
|
||||||
|
uq_source_artist_platform_url and uq_post_source_external_id.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0021"
|
||||||
|
down_revision: Union[str, None] = "0020"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM image_provenance ip1
|
||||||
|
USING image_provenance ip2
|
||||||
|
WHERE ip1.image_record_id = ip2.image_record_id
|
||||||
|
AND ip1.post_id = ip2.post_id
|
||||||
|
AND ip1.id > ip2.id
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_image_provenance_image_post",
|
||||||
|
"image_provenance",
|
||||||
|
["image_record_id", "post_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint(
|
||||||
|
"uq_image_provenance_image_post",
|
||||||
|
"image_provenance",
|
||||||
|
type_="unique",
|
||||||
|
)
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources
|
||||||
|
|
||||||
|
Revision ID: 0022
|
||||||
|
Revises: 0021
|
||||||
|
Create Date: 2026-05-26
|
||||||
|
|
||||||
|
Closes the operator-flagged 2026-05-26 issue where the filesystem importer
|
||||||
|
called _find_or_create_source(url=sd.post_url), creating one Source row per
|
||||||
|
imported post URL. Operator's Atole artist had 406 Source rows where there
|
||||||
|
should have been 1 (the /cw/Atole subscription Source).
|
||||||
|
|
||||||
|
Source represents a subscription feed (one per artist+platform — the
|
||||||
|
gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The
|
||||||
|
filesystem importer was misusing Source as a per-post key.
|
||||||
|
|
||||||
|
Migration steps per (artist_id, platform) group with >1 Source:
|
||||||
|
1. Pick canonical — prefer a URL NOT matching '/posts/<id>$' (real
|
||||||
|
campaign URL like /cw/Atole); else min(id).
|
||||||
|
2. PRE-merge any Posts under non-canonical sources whose
|
||||||
|
external_post_id ALREADY exists under the canonical source. (Same
|
||||||
|
gallery-dl post imported via two different sidecar paths can plant
|
||||||
|
two Post rows with identical external_post_id under different
|
||||||
|
Sources for the same artist.) Repoint ImageProvenance +
|
||||||
|
ImageRecord.primary_post_id to the canonical-side Post, dedupe
|
||||||
|
ImageProvenance against alembic 0021's uq, then delete the
|
||||||
|
non-canonical-side Post. This MUST happen before step 3 — Postgres
|
||||||
|
fires uq_post_source_external_id row-by-row during the bulk UPDATE
|
||||||
|
and the merge-after-reparent ordering 500s on first collision
|
||||||
|
(operator-hit during v26.05.26.1 deploy, 2026-05-26).
|
||||||
|
3. Reparent remaining Posts onto canonical (no collisions possible now).
|
||||||
|
4. Reparent ImageProvenance.source_id off the non-canonical sources.
|
||||||
|
5. Delete the orphan Source rows.
|
||||||
|
6. If the canonical Source's URL still looks like a per-post URL (no
|
||||||
|
campaign URL existed among candidates), rewrite it to
|
||||||
|
'sidecar:<platform>:<artist_slug>' so the artist detail page shows
|
||||||
|
something readable.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0022"
|
||||||
|
down_revision: Union[str, None] = "0021"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
_POST_URL_RE = r"/posts/[^/]+$"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
|
||||||
|
# Find (artist_id, platform) groups with > 1 Source row.
|
||||||
|
groups = conn.execute(text("""
|
||||||
|
SELECT artist_id, platform
|
||||||
|
FROM source
|
||||||
|
GROUP BY artist_id, platform
|
||||||
|
HAVING COUNT(*) > 1
|
||||||
|
""")).fetchall()
|
||||||
|
|
||||||
|
for artist_id, platform in groups:
|
||||||
|
rows = conn.execute(
|
||||||
|
text("""
|
||||||
|
SELECT id, url FROM source
|
||||||
|
WHERE artist_id = :a AND platform = :p
|
||||||
|
ORDER BY id ASC
|
||||||
|
"""),
|
||||||
|
{"a": artist_id, "p": platform},
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
# Canonical: first row whose URL doesn't look like a per-post URL;
|
||||||
|
# else min(id).
|
||||||
|
canonical_id = None
|
||||||
|
for sid, url in rows:
|
||||||
|
if not _matches_post_url(url):
|
||||||
|
canonical_id = sid
|
||||||
|
break
|
||||||
|
if canonical_id is None:
|
||||||
|
canonical_id = rows[0][0]
|
||||||
|
|
||||||
|
other_ids = [sid for sid, _ in rows if sid != canonical_id]
|
||||||
|
if not other_ids:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# STEP 2: PRE-merge ALL Posts with duplicate external_post_id
|
||||||
|
# across the entire (canonical + others) group, BEFORE the bulk
|
||||||
|
# reparent. Two cases must both be handled:
|
||||||
|
# (A) canonical has Post X with epid=N; an "other" source has
|
||||||
|
# Post Y with epid=N → after bulk UPDATE, (canonical, N)
|
||||||
|
# collides with itself.
|
||||||
|
# (B) two different "other" sources each have a Post with
|
||||||
|
# epid=N; canonical has none → after bulk UPDATE, both
|
||||||
|
# are repointed to (canonical, N) and the second collides.
|
||||||
|
# The earlier version of this migration only handled (A); the
|
||||||
|
# operator's deploy 2026-05-26 tripped (B) at line 139.
|
||||||
|
# Fix: group ALL Posts in the (artist, platform) by epid; for
|
||||||
|
# any group with count>1, pick the keep (prefer one already
|
||||||
|
# under canonical; else lowest id) and merge the rest into it.
|
||||||
|
all_posts = conn.execute(
|
||||||
|
text("""
|
||||||
|
SELECT external_post_id, id, source_id
|
||||||
|
FROM post
|
||||||
|
WHERE source_id = :canonical OR source_id = ANY(:others)
|
||||||
|
ORDER BY external_post_id, id
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "others": other_ids},
|
||||||
|
).fetchall()
|
||||||
|
by_epid: dict = {}
|
||||||
|
for epid, post_id, src_id in all_posts:
|
||||||
|
by_epid.setdefault(epid, []).append((post_id, src_id))
|
||||||
|
for _epid, posts in by_epid.items():
|
||||||
|
if len(posts) <= 1:
|
||||||
|
continue
|
||||||
|
# Prefer a Post already under canonical as the keep.
|
||||||
|
canonical_posts = [p for p in posts if p[1] == canonical_id]
|
||||||
|
if canonical_posts:
|
||||||
|
keep_id = canonical_posts[0][0]
|
||||||
|
else:
|
||||||
|
keep_id = posts[0][0] # already sorted by id ASC
|
||||||
|
drop_ids = [p[0] for p in posts if p[0] != keep_id]
|
||||||
|
for drop_id in drop_ids:
|
||||||
|
# Pre-delete image_provenance rows under drop_ whose
|
||||||
|
# image_record_id ALREADY has a provenance under keep —
|
||||||
|
# the UPDATE below would otherwise repoint them and
|
||||||
|
# trip uq_image_provenance_image_post (alembic 0021)
|
||||||
|
# row-by-row before any after-the-fact dedupe could
|
||||||
|
# run. Operator's v26.05.26.3 deploy 2026-05-26 tripped
|
||||||
|
# this at line 123.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
DELETE FROM image_provenance
|
||||||
|
WHERE post_id = :drop_
|
||||||
|
AND image_record_id IN (
|
||||||
|
SELECT image_record_id FROM image_provenance
|
||||||
|
WHERE post_id = :keep
|
||||||
|
)
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
# Now safe to repoint the survivors.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_provenance SET post_id = :keep
|
||||||
|
WHERE post_id = :drop_
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_record SET primary_post_id = :keep
|
||||||
|
WHERE primary_post_id = :drop_
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
text("DELETE FROM post WHERE id = :drop_"),
|
||||||
|
{"drop_": drop_id},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP 3: Bulk reparent the remaining Posts off the other
|
||||||
|
# Sources. After step 2, no collisions on
|
||||||
|
# (canonical, external_post_id) are possible.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE post SET source_id = :canonical
|
||||||
|
WHERE source_id = ANY(:others)
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "others": other_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP 4: Reparent ImageProvenance.source_id (denormalized FK).
|
||||||
|
# No UNIQUE on source_id; safe bulk update.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_provenance SET source_id = :canonical
|
||||||
|
WHERE source_id = ANY(:others)
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "others": other_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP 5: Drop the orphan Sources.
|
||||||
|
conn.execute(
|
||||||
|
text("DELETE FROM source WHERE id = ANY(:others)"),
|
||||||
|
{"others": other_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# If the canonical's URL still looks per-post (no campaign URL
|
||||||
|
# existed among the candidates), rewrite to a synthetic anchor so
|
||||||
|
# the artist detail page renders something readable.
|
||||||
|
canonical_url = conn.execute(
|
||||||
|
text("SELECT url FROM source WHERE id = :id"),
|
||||||
|
{"id": canonical_id},
|
||||||
|
).scalar_one()
|
||||||
|
if _matches_post_url(canonical_url):
|
||||||
|
slug = conn.execute(
|
||||||
|
text("SELECT slug FROM artist WHERE id = :id"),
|
||||||
|
{"id": artist_id},
|
||||||
|
).scalar_one()
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE source
|
||||||
|
SET url = :new_url, enabled = false
|
||||||
|
WHERE id = :id
|
||||||
|
"""),
|
||||||
|
{
|
||||||
|
"id": canonical_id,
|
||||||
|
"new_url": f"sidecar:{platform}:{slug}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Lossy migration — orphan Sources deleted, Posts reparented, Posts
|
||||||
|
# merged. No safe downgrade. If you need to roll back the schema
|
||||||
|
# invariant, fork from 0021 and re-run filesystem imports.
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _matches_post_url(url: str) -> bool:
|
||||||
|
"""True if url ends with /posts/<token> (gallery-dl-style per-post URL)."""
|
||||||
|
import re
|
||||||
|
return bool(re.search(_POST_URL_RE, url or ""))
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
"""drop meta + rating tag kinds — operator-retired 2026-05-26
|
||||||
|
|
||||||
|
Revision ID: 0023
|
||||||
|
Revises: 0022
|
||||||
|
Create Date: 2026-05-26
|
||||||
|
|
||||||
|
Operator decided meta + rating aren't valid tag kinds for FC. Per-row
|
||||||
|
behavior: DELETE existing rows (operator chose "clean break" over
|
||||||
|
"convert to general"). All cascading FKs (image_tag, tag_alias,
|
||||||
|
tag_allowlist, tag_reference_embedding, tag_suggestion_rejection,
|
||||||
|
series_page) use ondelete="CASCADE" so a single DELETE on tag cleans
|
||||||
|
the related rows in one go.
|
||||||
|
|
||||||
|
After the data cleanup, recreate the tag_kind ENUM without 'meta' /
|
||||||
|
'rating' (Postgres has no `ALTER TYPE ... DROP VALUE`; standard
|
||||||
|
rename-create-cast-drop dance). The server default 'general' is
|
||||||
|
dropped before the type swap and restored after.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0023"
|
||||||
|
down_revision: Union[str, None] = "0022"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# 1. Delete tags of the retired kinds. CASCADE handles related tables.
|
||||||
|
op.execute("DELETE FROM tag WHERE kind IN ('meta', 'rating')")
|
||||||
|
|
||||||
|
# 2. Drop the CHECK constraint that references the enum's literal
|
||||||
|
# values. Postgres can't resolve `kind = 'character'` across the
|
||||||
|
# type swap below — the literal would bind to the new tag_kind
|
||||||
|
# but the column is on tag_kind_old, producing
|
||||||
|
# "operator does not exist: tag_kind = tag_kind_old".
|
||||||
|
# (Operator-hit during the v26.05.26.5 deploy attempt; ck was
|
||||||
|
# originally added by alembic 0002.) Recreated post-swap.
|
||||||
|
op.drop_constraint(
|
||||||
|
"ck_tag_fandom_requires_character", "tag", type_="check"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. Drop the server default — ALTER COLUMN TYPE can't carry it
|
||||||
|
# across the type swap below.
|
||||||
|
op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
|
||||||
|
|
||||||
|
# 4. Recreate the tag_kind enum without meta/rating.
|
||||||
|
op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
|
||||||
|
op.execute(
|
||||||
|
"CREATE TYPE tag_kind AS ENUM ("
|
||||||
|
"'artist', 'character', 'fandom', 'general', "
|
||||||
|
"'series', 'archive', 'post'"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"ALTER TABLE tag "
|
||||||
|
"ALTER COLUMN kind TYPE tag_kind "
|
||||||
|
"USING kind::text::tag_kind"
|
||||||
|
)
|
||||||
|
op.execute("DROP TYPE tag_kind_old")
|
||||||
|
|
||||||
|
# 5. Restore the server default.
|
||||||
|
op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
|
||||||
|
|
||||||
|
# 6. Restore the CHECK constraint (now bound to the new tag_kind).
|
||||||
|
op.create_check_constraint(
|
||||||
|
"ck_tag_fandom_requires_character",
|
||||||
|
"tag",
|
||||||
|
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Add the values back to the enum so old code can boot. The deleted
|
||||||
|
# tag rows are gone permanently — no safe restore.
|
||||||
|
op.drop_constraint(
|
||||||
|
"ck_tag_fandom_requires_character", "tag", type_="check"
|
||||||
|
)
|
||||||
|
op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT")
|
||||||
|
op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old")
|
||||||
|
op.execute(
|
||||||
|
"CREATE TYPE tag_kind AS ENUM ("
|
||||||
|
"'artist', 'character', 'fandom', 'general', "
|
||||||
|
"'series', 'archive', 'post', 'meta', 'rating'"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"ALTER TABLE tag "
|
||||||
|
"ALTER COLUMN kind TYPE tag_kind "
|
||||||
|
"USING kind::text::tag_kind"
|
||||||
|
)
|
||||||
|
op.execute("DROP TYPE tag_kind_old")
|
||||||
|
op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'")
|
||||||
|
op.create_check_constraint(
|
||||||
|
"ck_tag_fandom_requires_character",
|
||||||
|
"tag",
|
||||||
|
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||||
|
)
|
||||||
+38
-1
@@ -3,13 +3,23 @@
|
|||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from quart import Quart
|
from quart import Quart, request
|
||||||
|
|
||||||
from .api import all_blueprints
|
from .api import all_blueprints
|
||||||
from .config import get_config
|
from .config import get_config
|
||||||
from .frontend import frontend_bp
|
from .frontend import frontend_bp
|
||||||
from .services.credential_crypto import CredentialCrypto
|
from .services.credential_crypto import CredentialCrypto
|
||||||
|
|
||||||
|
# Browser-extension origins. The FabledCurator extension fetches from
|
||||||
|
# moz-extension://<uuid>/ on Firefox and chrome-extension://<uuid>/ on
|
||||||
|
# Chromium-based browsers. Operator-flagged 2026-05-26: extension's
|
||||||
|
# 'Test connection' returned `NetworkError` because the X-Extension-Key
|
||||||
|
# header on /api/credentials triggers a CORS preflight that our routes
|
||||||
|
# don't handle. Whitelisting only these two schemes (not opening CORS
|
||||||
|
# up generally) lets the extension talk to a plain-HTTP self-hosted FC
|
||||||
|
# without weakening the no-CORS posture for normal browser usage.
|
||||||
|
_EXTENSION_ORIGIN_SCHEMES = ("moz-extension://", "chrome-extension://")
|
||||||
|
|
||||||
_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||||
|
|
||||||
|
|
||||||
@@ -35,6 +45,33 @@ def create_app() -> Quart:
|
|||||||
# Registered last so /api/* routes win over the SPA catch-all.
|
# Registered last so /api/* routes win over the SPA catch-all.
|
||||||
app.register_blueprint(frontend_bp)
|
app.register_blueprint(frontend_bp)
|
||||||
|
|
||||||
|
@app.before_request
|
||||||
|
async def _extension_cors_preflight():
|
||||||
|
# Short-circuit OPTIONS preflight from the browser extension with a
|
||||||
|
# 204 + CORS headers (the after_request hook below adds them).
|
||||||
|
# Without this, OPTIONS lands on routes that only declared POST/GET
|
||||||
|
# methods and 405s before the after_request gets a chance.
|
||||||
|
if request.method != "OPTIONS":
|
||||||
|
return None
|
||||||
|
origin = request.headers.get("Origin", "")
|
||||||
|
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||||
|
return "", 204
|
||||||
|
return None
|
||||||
|
|
||||||
|
@app.after_request
|
||||||
|
async def _extension_cors_headers(response):
|
||||||
|
origin = request.headers.get("Origin", "")
|
||||||
|
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||||
|
response.headers["Access-Control-Allow-Origin"] = origin
|
||||||
|
response.headers["Access-Control-Allow-Methods"] = (
|
||||||
|
"GET, POST, PATCH, DELETE, OPTIONS"
|
||||||
|
)
|
||||||
|
response.headers["Access-Control-Allow-Headers"] = (
|
||||||
|
"Content-Type, X-Extension-Key"
|
||||||
|
)
|
||||||
|
response.headers["Access-Control-Max-Age"] = "86400"
|
||||||
|
return response
|
||||||
|
|
||||||
@app.after_serving
|
@app.after_serving
|
||||||
async def _dispose_db_engine() -> None:
|
async def _dispose_db_engine() -> None:
|
||||||
from .extensions import dispose_engine
|
from .extensions import dispose_engine
|
||||||
|
|||||||
@@ -14,11 +14,13 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
|
|||||||
|
|
||||||
|
|
||||||
def all_blueprints() -> list[Blueprint]:
|
def all_blueprints() -> list[Blueprint]:
|
||||||
|
from .admin import admin_bp
|
||||||
from .aliases import aliases_bp
|
from .aliases import aliases_bp
|
||||||
from .allowlist import allowlist_bp
|
from .allowlist import allowlist_bp
|
||||||
from .artist import artist_bp
|
from .artist import artist_bp
|
||||||
from .artists import artists_bp
|
from .artists import artists_bp
|
||||||
from .attachments import attachments_bp
|
from .attachments import attachments_bp
|
||||||
|
from .cleanup import cleanup_bp
|
||||||
from .credentials import credentials_bp
|
from .credentials import credentials_bp
|
||||||
from .downloads import downloads_bp
|
from .downloads import downloads_bp
|
||||||
from .extension import extension_bp
|
from .extension import extension_bp
|
||||||
@@ -33,7 +35,10 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .showcase import showcase_bp
|
from .showcase import showcase_bp
|
||||||
from .sources import sources_bp
|
from .sources import sources_bp
|
||||||
from .suggestions import suggestions_bp
|
from .suggestions import suggestions_bp
|
||||||
|
from .system_activity import system_activity_bp
|
||||||
|
from .system_backup import system_backup_bp
|
||||||
from .tags import tags_bp
|
from .tags import tags_bp
|
||||||
|
from .thumbnails import thumbnails_bp
|
||||||
return [
|
return [
|
||||||
api_bp,
|
api_bp,
|
||||||
attachments_bp,
|
attachments_bp,
|
||||||
@@ -44,12 +49,17 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
artists_bp,
|
artists_bp,
|
||||||
showcase_bp,
|
showcase_bp,
|
||||||
settings_bp,
|
settings_bp,
|
||||||
|
system_activity_bp,
|
||||||
|
system_backup_bp,
|
||||||
|
admin_bp,
|
||||||
|
cleanup_bp,
|
||||||
import_admin_bp,
|
import_admin_bp,
|
||||||
migrate_bp,
|
migrate_bp,
|
||||||
suggestions_bp,
|
suggestions_bp,
|
||||||
allowlist_bp,
|
allowlist_bp,
|
||||||
aliases_bp,
|
aliases_bp,
|
||||||
ml_admin_bp,
|
ml_admin_bp,
|
||||||
|
thumbnails_bp,
|
||||||
sources_bp,
|
sources_bp,
|
||||||
platforms_bp,
|
platforms_bp,
|
||||||
posts_bp,
|
posts_bp,
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
"""FC-3k: /api/admin — destructive admin actions.
|
||||||
|
|
||||||
|
Five action surfaces:
|
||||||
|
POST /api/admin/artists/<slug>/cascade-delete (Tier C)
|
||||||
|
POST /api/admin/images/bulk-delete (Tier C)
|
||||||
|
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||||
|
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||||
|
POST /api/admin/tags/prune-unused (Tier A)
|
||||||
|
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||||
|
|
||||||
|
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||||
|
no dispatch) and a confirm body field (server-recomputed token).
|
||||||
|
Long-running ops dispatch a maintenance-queue Celery task; the UI
|
||||||
|
tails FC-3i's /api/system/activity/runs to surface progress.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import Artist
|
||||||
|
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
||||||
|
|
||||||
|
admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
|
||||||
|
|
||||||
|
|
||||||
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
|
body = {"error": error}
|
||||||
|
body.update(extra)
|
||||||
|
return jsonify(body), status
|
||||||
|
|
||||||
|
|
||||||
|
def _bulk_image_confirm_token(image_ids: list[int]) -> str:
|
||||||
|
"""Stable 8-hex token derived from the sorted id list. Mutates
|
||||||
|
when the selection changes; stays the same across modal opens of
|
||||||
|
the same selection so the operator can paste without confusion."""
|
||||||
|
canon = ",".join(str(i) for i in sorted(image_ids))
|
||||||
|
digest = hashlib.sha256(canon.encode("utf-8")).hexdigest()
|
||||||
|
return digest[:8]
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/artists/<slug>/cascade-delete", methods=["POST"])
|
||||||
|
async def artist_cascade_delete(slug: str):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
dry_run = bool(body.get("dry_run", False))
|
||||||
|
supplied_confirm = body.get("confirm", "")
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
artist = (await session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slug)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
artist_id = artist.id
|
||||||
|
|
||||||
|
projected = await session.run_sync(
|
||||||
|
lambda sync_sess: project_artist_cascade(sync_sess, slug=slug)
|
||||||
|
)
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return jsonify(projected)
|
||||||
|
|
||||||
|
expected = f"delete-artist-{artist_id}"
|
||||||
|
if supplied_confirm != expected:
|
||||||
|
return _bad(
|
||||||
|
"confirm_mismatch",
|
||||||
|
detail=f"confirm must equal {expected!r}",
|
||||||
|
expected=expected,
|
||||||
|
)
|
||||||
|
|
||||||
|
from ..tasks.admin import delete_artist_cascade_task
|
||||||
|
async_result = delete_artist_cascade_task.delay(artist_id=artist_id)
|
||||||
|
return jsonify({"task_id": async_result.id}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/images/bulk-delete", methods=["POST"])
|
||||||
|
async def images_bulk_delete():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
image_ids = body.get("image_ids")
|
||||||
|
if not isinstance(image_ids, list) or not image_ids:
|
||||||
|
return _bad("invalid_image_ids", detail="image_ids must be non-empty list of int")
|
||||||
|
try:
|
||||||
|
image_ids = [int(i) for i in image_ids]
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_image_ids", detail="image_ids must contain only ints")
|
||||||
|
|
||||||
|
dry_run = bool(body.get("dry_run", False))
|
||||||
|
supplied_confirm = body.get("confirm", "")
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
projected = await session.run_sync(
|
||||||
|
lambda sync_sess: project_bulk_image_delete(
|
||||||
|
sync_sess, image_ids=image_ids,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return jsonify(projected)
|
||||||
|
|
||||||
|
sha8 = _bulk_image_confirm_token(image_ids)
|
||||||
|
expected = f"delete-images-{sha8}"
|
||||||
|
if supplied_confirm != expected:
|
||||||
|
return _bad(
|
||||||
|
"confirm_mismatch",
|
||||||
|
detail=f"confirm must equal {expected!r}",
|
||||||
|
expected=expected,
|
||||||
|
)
|
||||||
|
|
||||||
|
from ..tasks.admin import bulk_delete_images_task
|
||||||
|
async_result = bulk_delete_images_task.delay(image_ids=image_ids)
|
||||||
|
return jsonify({"task_id": async_result.id}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/<int:tag_id>", methods=["DELETE"])
|
||||||
|
async def tag_delete(tag_id: int):
|
||||||
|
"""Tier-B sync delete. UI yes/no modal is the only confirmation."""
|
||||||
|
from ..services.cleanup_service import delete_tag
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
result = await session.run_sync(
|
||||||
|
lambda sync_sess: delete_tag(sync_sess, tag_id=tag_id)
|
||||||
|
)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/<int:dest_id>/merge", methods=["POST"])
|
||||||
|
async def tag_merge(dest_id: int):
|
||||||
|
"""Wraps TagService.merge. Source repoints to dest, dest survives,
|
||||||
|
source row deleted, protective alias auto-created if source was
|
||||||
|
ML-applied or allowlisted."""
|
||||||
|
from ..services.tag_service import TagMergeConflict, TagService, TagValidationError
|
||||||
|
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
source_id = body.get("source_id")
|
||||||
|
if not isinstance(source_id, int) or source_id == dest_id:
|
||||||
|
return _bad("invalid_source_id", detail="source_id must be int and differ from dest")
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
result = await TagService(session).merge(
|
||||||
|
source_id=source_id, target_id=dest_id,
|
||||||
|
)
|
||||||
|
except TagMergeConflict as exc:
|
||||||
|
return _bad("merge_conflict", status=409, detail=str(exc))
|
||||||
|
except TagValidationError as exc:
|
||||||
|
return _bad("tag_kind_mismatch", detail=str(exc))
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
|
||||||
|
# MergeResult is a frozen dataclass — flatten to dict.
|
||||||
|
return jsonify({
|
||||||
|
"result": {
|
||||||
|
"target_id": result.target_id,
|
||||||
|
"target_name": result.target_name,
|
||||||
|
"target_kind": result.target_kind,
|
||||||
|
"merged_count": result.merged_count,
|
||||||
|
"alias_created": result.alias_created,
|
||||||
|
"source_deleted": result.source_deleted,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/<int:tag_id>/usage-count", methods=["GET"])
|
||||||
|
async def tag_usage_count(tag_id: int):
|
||||||
|
"""Helper for the Tier-B yes/no prompt; surfaces "N associations"
|
||||||
|
in the dialog so the operator knows what they're nuking."""
|
||||||
|
from ..services.cleanup_service import count_tag_associations
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
count = await session.run_sync(
|
||||||
|
lambda sync_sess: count_tag_associations(
|
||||||
|
sync_sess, tag_id=tag_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return jsonify({"count": count})
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/prune-unused", methods=["POST"])
|
||||||
|
async def tags_prune_unused():
|
||||||
|
"""Tier-A: dry-run preview list IS the prompt. UI calls with
|
||||||
|
dry_run=true first, shows the list, operator clicks button to
|
||||||
|
re-call with dry_run=false."""
|
||||||
|
from ..services.cleanup_service import prune_unused_tags
|
||||||
|
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
dry_run = bool(body.get("dry_run", False))
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(
|
||||||
|
lambda sync_sess: prune_unused_tags(
|
||||||
|
sync_sess, dry_run=dry_run,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return jsonify(result)
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
"""FC-Cleanup: /api/cleanup/* — retroactive enforcement of import filters.
|
||||||
|
|
||||||
|
Endpoints:
|
||||||
|
POST /min-dimension/preview synchronous SQL audit
|
||||||
|
POST /min-dimension/delete synchronous SQL delete (Tier-C token)
|
||||||
|
POST /audit async transparency / single_color start
|
||||||
|
GET /audit list recent audit_run rows
|
||||||
|
GET /audit/<id> single audit_run row
|
||||||
|
POST /audit/<id>/apply apply matched_ids deletes (Tier-C token)
|
||||||
|
POST /audit/<id>/cancel flip running audit to cancelled
|
||||||
|
|
||||||
|
Unused-tags retroactive prune intentionally NOT in this namespace —
|
||||||
|
TagMaintenanceCard (Maintenance tab → moved to Cleanup tab in v26.05.25.7)
|
||||||
|
uses the existing /api/admin/tags/prune-unused endpoint via the admin
|
||||||
|
store. No duplicate route here.
|
||||||
|
|
||||||
|
Confirm-token format matches modal/DestructiveConfirmModal.vue convention:
|
||||||
|
`delete-min-dim-<sha8(w,h)>` for min-dim delete
|
||||||
|
`delete-audit-<id>` for audit apply
|
||||||
|
(Modal hardcodes action ∈ {'restore', 'delete'}; "apply audit" is semantically a delete of the matched images, so we use `delete-audit-<id>`.)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import LibraryAuditRun
|
||||||
|
from ..services import cleanup_service
|
||||||
|
|
||||||
|
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
||||||
|
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
|
body = {"error": error}
|
||||||
|
body.update(extra)
|
||||||
|
return jsonify(body), status
|
||||||
|
|
||||||
|
|
||||||
|
def _min_dim_token(min_w: int, min_h: int) -> str:
|
||||||
|
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
||||||
|
# sides use SHA-256 truncated to 8 hex chars.
|
||||||
|
canon = f"{min_w}x{min_h}"
|
||||||
|
return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}"
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_audit_run(audit: LibraryAuditRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": audit.id,
|
||||||
|
"rule": audit.rule,
|
||||||
|
"params": audit.params,
|
||||||
|
"status": audit.status,
|
||||||
|
"started_at": audit.started_at.isoformat() if audit.started_at else None,
|
||||||
|
"finished_at": audit.finished_at.isoformat() if audit.finished_at else None,
|
||||||
|
"scanned_count": audit.scanned_count,
|
||||||
|
"matched_count": audit.matched_count,
|
||||||
|
"matched_ids": audit.matched_ids,
|
||||||
|
"error": audit.error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/min-dimension/preview", methods=["POST"])
|
||||||
|
async def min_dim_preview():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
min_w = int(body.get("min_width", 0))
|
||||||
|
min_h = int(body.get("min_height", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
if min_w < 0 or min_h < 0:
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
async with get_session() as session:
|
||||||
|
projection = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.project_min_dimension_violations(
|
||||||
|
s, min_width=min_w, min_height=min_h,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return jsonify(projection)
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/min-dimension/delete", methods=["POST"])
|
||||||
|
async def min_dim_delete():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
min_w = int(body.get("min_width", 0))
|
||||||
|
min_h = int(body.get("min_height", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
if min_w < 0 or min_h < 0:
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
supplied = body.get("confirm", "")
|
||||||
|
expected = _min_dim_token(min_w, min_h)
|
||||||
|
if supplied != expected:
|
||||||
|
return _bad("confirm_mismatch", expected=expected)
|
||||||
|
async with get_session() as session:
|
||||||
|
deleted = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.delete_min_dimension_violations(
|
||||||
|
s, min_width=min_w, min_height=min_h, images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"deleted": deleted})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit", methods=["POST"])
|
||||||
|
async def audit_create():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
rule = body.get("rule")
|
||||||
|
params = body.get("params") or {}
|
||||||
|
if rule not in ("transparency", "single_color"):
|
||||||
|
return _bad("invalid_rule")
|
||||||
|
if not isinstance(params, dict):
|
||||||
|
return _bad("invalid_params")
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
audit_id = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.start_audit_run(
|
||||||
|
s, rule=rule, params=params,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except cleanup_service.AuditAlreadyRunning as running_id:
|
||||||
|
return _bad(
|
||||||
|
"audit_already_running", status=409,
|
||||||
|
running_id=int(str(running_id)),
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return _bad(str(exc))
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"audit_id": audit_id, "status": "running"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>", methods=["GET"])
|
||||||
|
async def audit_get(audit_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
audit = (await session.execute(
|
||||||
|
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if audit is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
return jsonify(_serialize_audit_run(audit))
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit", methods=["GET"])
|
||||||
|
async def audit_history():
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "20")), 100)
|
||||||
|
except ValueError:
|
||||||
|
return _bad("invalid_limit")
|
||||||
|
async with get_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(LibraryAuditRun)
|
||||||
|
.order_by(LibraryAuditRun.id.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)).scalars().all()
|
||||||
|
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>/apply", methods=["POST"])
|
||||||
|
async def audit_apply(audit_id: int):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
confirm = body.get("confirm", "")
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
deleted = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.apply_audit_run(
|
||||||
|
s, audit_id=audit_id, confirm_token=confirm,
|
||||||
|
images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except cleanup_service.AuditNotReady as exc:
|
||||||
|
return _bad("audit_not_ready", current_status=str(exc))
|
||||||
|
except cleanup_service.ConfirmTokenMismatch as exc:
|
||||||
|
return _bad("confirm_mismatch", expected=str(exc))
|
||||||
|
except ValueError as exc:
|
||||||
|
return _bad("not_found", status=404, detail=str(exc))
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"deleted": deleted})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>/cancel", methods=["POST"])
|
||||||
|
async def audit_cancel(audit_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
await session.run_sync(
|
||||||
|
lambda s: cleanup_service.cancel_audit_run(s, audit_id=audit_id)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"cancelled": True})
|
||||||
@@ -93,10 +93,20 @@ def _read_manifest_sync() -> dict | None:
|
|||||||
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
||||||
if not XPI_DIR.is_dir():
|
if not XPI_DIR.is_dir():
|
||||||
return None
|
return None
|
||||||
xpis = sorted(XPI_DIR.glob("fabledcurator-*.xpi"), key=lambda p: p.stat().st_mtime)
|
# Exclude the `fabledcurator-latest.xpi` alias when picking the file to
|
||||||
if not xpis:
|
# extract a version from — it's a copy of the latest versioned XPI,
|
||||||
|
# written at the same mtime by build.yml, and would otherwise tie or
|
||||||
|
# win the sort (operator-flagged 2026-05-26: UI displayed "v latest"
|
||||||
|
# because `_extract_version("fabledcurator-latest.xpi")` returns
|
||||||
|
# the literal "latest"). The alias still serves as `latest_url`.
|
||||||
|
versioned = [
|
||||||
|
p for p in XPI_DIR.glob("fabledcurator-*.xpi")
|
||||||
|
if p.name != "fabledcurator-latest.xpi"
|
||||||
|
]
|
||||||
|
if not versioned:
|
||||||
return None
|
return None
|
||||||
latest = xpis[-1]
|
versioned.sort(key=lambda p: p.stat().st_mtime)
|
||||||
|
latest = versioned[-1]
|
||||||
return {
|
return {
|
||||||
"installed": True,
|
"installed": True,
|
||||||
"version": _extract_version(latest.name),
|
"version": _extract_version(latest.name),
|
||||||
|
|||||||
@@ -42,6 +42,8 @@ async def scroll():
|
|||||||
"width": i.width,
|
"width": i.width,
|
||||||
"height": i.height,
|
"height": i.height,
|
||||||
"created_at": i.created_at.isoformat(),
|
"created_at": i.created_at.isoformat(),
|
||||||
|
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
|
||||||
|
"effective_date": i.effective_date.isoformat(),
|
||||||
"thumbnail_url": i.thumbnail_url,
|
"thumbnail_url": i.thumbnail_url,
|
||||||
"artist": i.artist,
|
"artist": i.artist,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,10 +47,13 @@ async def status():
|
|||||||
if active:
|
if active:
|
||||||
payload["active_batch"] = {
|
payload["active_batch"] = {
|
||||||
"id": active.id,
|
"id": active.id,
|
||||||
|
"source_path": active.source_path,
|
||||||
|
"scan_mode": active.scan_mode,
|
||||||
"total_files": active.total_files,
|
"total_files": active.total_files,
|
||||||
"imported": active.imported,
|
"imported": active.imported,
|
||||||
"skipped": active.skipped,
|
"skipped": active.skipped,
|
||||||
"failed": active.failed,
|
"failed": active.failed,
|
||||||
|
"refreshed": active.refreshed,
|
||||||
"started_at": active.started_at.isoformat(),
|
"started_at": active.started_at.isoformat(),
|
||||||
}
|
}
|
||||||
return jsonify(payload)
|
return jsonify(payload)
|
||||||
@@ -100,17 +103,22 @@ async def list_tasks():
|
|||||||
|
|
||||||
@import_admin_bp.route("/retry-failed", methods=["POST"])
|
@import_admin_bp.route("/retry-failed", methods=["POST"])
|
||||||
async def retry_failed():
|
async def retry_failed():
|
||||||
|
# Fold SELECT into UPDATE…WHERE…RETURNING — the prior SELECT-then-
|
||||||
|
# UPDATE-WHERE-id-IN pattern blew past psycopg's 65535-parameter
|
||||||
|
# ceiling once failed_ids exceeded ~65k rows.
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
failed_ids = (
|
result = await session.execute(
|
||||||
await session.execute(select(ImportTask.id).where(ImportTask.status == "failed"))
|
update(ImportTask)
|
||||||
).scalars().all()
|
.where(ImportTask.status == "failed")
|
||||||
|
.values(
|
||||||
|
status="queued", error=None,
|
||||||
|
started_at=None, finished_at=None,
|
||||||
|
)
|
||||||
|
.returning(ImportTask.id)
|
||||||
|
)
|
||||||
|
failed_ids = [row[0] for row in result.all()]
|
||||||
if not failed_ids:
|
if not failed_ids:
|
||||||
return jsonify({"retried": 0})
|
return jsonify({"retried": 0})
|
||||||
await session.execute(
|
|
||||||
update(ImportTask)
|
|
||||||
.where(ImportTask.id.in_(failed_ids))
|
|
||||||
.values(status="queued", error=None, started_at=None, finished_at=None)
|
|
||||||
)
|
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
from ..tasks.import_file import import_media_file
|
from ..tasks.import_file import import_media_file
|
||||||
@@ -120,6 +128,81 @@ async def retry_failed():
|
|||||||
return jsonify({"retried": len(failed_ids)})
|
return jsonify({"retried": len(failed_ids)})
|
||||||
|
|
||||||
|
|
||||||
|
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||||
|
async def clear_stuck():
|
||||||
|
"""Force any non-terminal ImportTask (status in pending/queued/
|
||||||
|
processing) to 'failed' AND finalize any ImportBatch that ends up
|
||||||
|
with no active children. Escape hatch for the operator when the
|
||||||
|
automatic recover_interrupted_tasks sweep keeps re-queueing the
|
||||||
|
same stuck row forever (e.g., underlying file is genuinely broken
|
||||||
|
and the import keeps OSError-looping at PIL load).
|
||||||
|
|
||||||
|
Idempotent + non-destructive: rows survive as 'failed' so the
|
||||||
|
Retry-Failed button can re-attempt them once whatever was broken
|
||||||
|
is fixed. Banked 2026-05-25 — operator hit 3 large PNGs that
|
||||||
|
autoretry-looped for 2 days after a corrupt-data PIL OSError.
|
||||||
|
"""
|
||||||
|
async with get_session() as session:
|
||||||
|
# Fold SELECT into UPDATE…WHERE — see /retry-failed for the
|
||||||
|
# 65535-parameter ceiling rationale. rowcount is enough here
|
||||||
|
# because we don't need the ids afterward (no .delay()).
|
||||||
|
clear_result = await session.execute(
|
||||||
|
update(ImportTask)
|
||||||
|
.where(
|
||||||
|
ImportTask.status.in_(["pending", "queued", "processing"])
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status="failed",
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
error=(
|
||||||
|
"manually cleared via /api/import/clear-stuck "
|
||||||
|
"— stuck in non-terminal state; retry once "
|
||||||
|
"underlying cause (corrupt file, missing model, "
|
||||||
|
"etc.) is resolved"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
tasks_failed = clear_result.rowcount or 0
|
||||||
|
|
||||||
|
# Finalize any 'running' ImportBatch that no longer has any
|
||||||
|
# active children. The "Scanning..." banner is driven by
|
||||||
|
# /api/import/status finding a running batch; left untouched,
|
||||||
|
# it would persist forever after the stuck-task clear.
|
||||||
|
running_batches = (
|
||||||
|
await session.execute(
|
||||||
|
select(ImportBatch.id).where(ImportBatch.status == "running")
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
finalized_batches = 0
|
||||||
|
for batch_id in running_batches:
|
||||||
|
still_active = (
|
||||||
|
await session.execute(
|
||||||
|
select(ImportTask.id)
|
||||||
|
.where(ImportTask.batch_id == batch_id)
|
||||||
|
.where(ImportTask.status.in_(
|
||||||
|
["pending", "queued", "processing"]
|
||||||
|
))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if still_active is None:
|
||||||
|
await session.execute(
|
||||||
|
update(ImportBatch)
|
||||||
|
.where(ImportBatch.id == batch_id)
|
||||||
|
.values(
|
||||||
|
status="complete",
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finalized_batches += 1
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
"tasks_failed": tasks_failed,
|
||||||
|
"batches_finalized": finalized_batches,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@import_admin_bp.route("/clear-completed", methods=["POST"])
|
@import_admin_bp.route("/clear-completed", methods=["POST"])
|
||||||
async def clear_completed():
|
async def clear_completed():
|
||||||
body = await request.get_json(silent=True) or {}
|
body = await request.get_json(silent=True) or {}
|
||||||
|
|||||||
@@ -1,14 +1,11 @@
|
|||||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
"""FC-5: /api/migrate — trigger and poll migration runs.
|
||||||
|
|
||||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
||||||
`export_file` field. All other kinds accept JSON. Apply-without-backup
|
`export_file` field. All other kinds accept JSON. Backup + rollback
|
||||||
guard rejects non-dry-run ingests unless a pre_migration-tagged backup
|
were retired in FC-3h (2026-05-24); use /api/system/backup/* instead.
|
||||||
exists in the last 24h (override with body.force=true).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from datetime import UTC, datetime, timedelta
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -19,12 +16,12 @@ from ..tasks.migration import run_migration
|
|||||||
|
|
||||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
||||||
|
|
||||||
|
# 'backup' + 'rollback' retired 2026-05-24 (FC-3h); see /api/system/backup/*.
|
||||||
_VALID_KINDS = frozenset({
|
_VALID_KINDS = frozenset({
|
||||||
"backup", "gs_ingest", "ir_ingest", "tag_apply",
|
"gs_ingest", "ir_ingest", "tag_apply",
|
||||||
"ml_queue", "verify", "rollback", "cleanup",
|
"ml_queue", "verify", "cleanup",
|
||||||
})
|
})
|
||||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
|
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
@@ -33,19 +30,6 @@ def _bad(error: str, *, status: int = 400, **extra):
|
|||||||
return jsonify(body), status
|
return jsonify(body), status
|
||||||
|
|
||||||
|
|
||||||
def _has_recent_pre_migration_backup() -> bool:
|
|
||||||
from ..services.migrators import backup as backup_mod
|
|
||||||
images_root = Path("/images")
|
|
||||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
|
||||||
if manifest is None:
|
|
||||||
return False
|
|
||||||
created_at_str = manifest.get("created_at")
|
|
||||||
if not created_at_str:
|
|
||||||
return False
|
|
||||||
created_at = datetime.fromisoformat(created_at_str)
|
|
||||||
return (datetime.now(UTC) - created_at) < timedelta(hours=24)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_to_dict(run: MigrationRun) -> dict:
|
def _run_to_dict(run: MigrationRun) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": run.id,
|
"id": run.id,
|
||||||
@@ -78,7 +62,6 @@ async def create_run(kind: str):
|
|||||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
return _bad("invalid_export_file", detail=str(exc))
|
return _bad("invalid_export_file", detail=str(exc))
|
||||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
||||||
force = str(form.get("force", "false")).lower() in ("true", "1", "yes")
|
|
||||||
params: dict = {"data": data, "dry_run": dry_run}
|
params: dict = {"data": data, "dry_run": dry_run}
|
||||||
else:
|
else:
|
||||||
body = await request.get_json()
|
body = await request.get_json()
|
||||||
@@ -87,20 +70,8 @@ async def create_run(kind: str):
|
|||||||
if not isinstance(body, dict):
|
if not isinstance(body, dict):
|
||||||
return _bad("invalid_body")
|
return _bad("invalid_body")
|
||||||
dry_run = bool(body.get("dry_run", False))
|
dry_run = bool(body.get("dry_run", False))
|
||||||
force = bool(body.get("force", False))
|
|
||||||
params = dict(body)
|
params = dict(body)
|
||||||
|
|
||||||
is_apply = (kind in _APPLY_KINDS) and not dry_run
|
|
||||||
if is_apply and not force and not _has_recent_pre_migration_backup():
|
|
||||||
return _bad(
|
|
||||||
"no_backup",
|
|
||||||
detail="apply action requires a pre_migration-tagged backup "
|
|
||||||
"in the last 24h (or force=true).",
|
|
||||||
)
|
|
||||||
|
|
||||||
if kind == "backup":
|
|
||||||
params.setdefault("tag", "pre_migration")
|
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
||||||
session.add(run)
|
session.add(run)
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""FC-3i: system activity dashboard endpoints.
|
||||||
|
|
||||||
|
Read-only. Combines Redis-broker queue depths (LLEN per queue),
|
||||||
|
Celery worker introspection (celery inspect), and the task_run DB
|
||||||
|
history into the surfaces the SystemActivityTab UI consumes.
|
||||||
|
|
||||||
|
All filesystem/sync-client work goes through asyncio.to_thread per
|
||||||
|
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import desc, func, select
|
||||||
|
|
||||||
|
from ..config import get_config
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import TaskRun
|
||||||
|
|
||||||
|
system_activity_bp = Blueprint(
|
||||||
|
"system_activity", __name__, url_prefix="/api/system/activity",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Canonical queue order — must match celery_app.task_routes. UI renders
|
||||||
|
# in this order; queues with no LLEN response show as null rather than
|
||||||
|
# absent.
|
||||||
|
_QUEUE_NAMES = (
|
||||||
|
"default", "import", "thumbnail", "ml",
|
||||||
|
"download", "scan", "maintenance",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Cache module-level so all requests share the cache between polls.
|
||||||
|
# Tests can reset via direct dict mutation if needed.
|
||||||
|
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
|
||||||
|
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
|
||||||
|
_QUEUE_CACHE_TTL = 2.0
|
||||||
|
_WORKER_CACHE_TTL = 5.0
|
||||||
|
|
||||||
|
|
||||||
|
def _read_queues_sync() -> dict:
|
||||||
|
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
|
||||||
|
asyncio.to_thread. Per-queue try/except returns None on failure so
|
||||||
|
one bad queue doesn't break the whole response."""
|
||||||
|
import redis # local import; only this endpoint needs it
|
||||||
|
|
||||||
|
cfg = get_config()
|
||||||
|
client = redis.Redis.from_url(cfg.celery_broker_url)
|
||||||
|
out: dict = {}
|
||||||
|
for name in _QUEUE_NAMES:
|
||||||
|
try:
|
||||||
|
out[name] = int(client.llen(name))
|
||||||
|
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
|
||||||
|
out[name] = None
|
||||||
|
return {
|
||||||
|
"queues": out,
|
||||||
|
"fetched_at": datetime.now(UTC).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _read_workers_sync() -> dict:
|
||||||
|
"""celery inspect active_queues + active. Returns per-worker info."""
|
||||||
|
from ..celery_app import celery as celery_app
|
||||||
|
|
||||||
|
insp = celery_app.control.inspect(timeout=2.0)
|
||||||
|
active_queues = insp.active_queues() or {}
|
||||||
|
active_tasks = insp.active() or {}
|
||||||
|
|
||||||
|
workers: dict = {}
|
||||||
|
for hostname, queues in active_queues.items():
|
||||||
|
workers[hostname] = {
|
||||||
|
"queues": sorted({q["name"] for q in queues}),
|
||||||
|
"active_count": len(active_tasks.get(hostname, [])),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"workers": workers,
|
||||||
|
"fetched_at": datetime.now(UTC).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/queues", methods=["GET"])
|
||||||
|
async def get_queues():
|
||||||
|
"""Per-queue Redis LLEN. Cached 2s.
|
||||||
|
|
||||||
|
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
||||||
|
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
||||||
|
_QUEUE_CACHE["ts"] = now
|
||||||
|
return jsonify(_QUEUE_CACHE["data"])
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/workers", methods=["GET"])
|
||||||
|
async def get_workers():
|
||||||
|
"""Live celery inspect. Cached 5s.
|
||||||
|
|
||||||
|
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
|
||||||
|
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
|
||||||
|
_WORKER_CACHE["ts"] = now
|
||||||
|
return jsonify(_WORKER_CACHE["data"])
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/runs", methods=["GET"])
|
||||||
|
async def list_runs():
|
||||||
|
"""Paginated task_run history. Query params:
|
||||||
|
queue=<name> filter to one queue
|
||||||
|
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||||
|
limit=<int> default 50, max 200
|
||||||
|
before_id=<int> cursor for keyset pagination
|
||||||
|
|
||||||
|
Response: {runs: [...], next_cursor: id|null}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "50")), 200)
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
if limit < 1:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
|
||||||
|
queue = request.args.get("queue")
|
||||||
|
status = request.args.get("status")
|
||||||
|
before_id_raw = request.args.get("before_id")
|
||||||
|
before_id = int(before_id_raw) if before_id_raw else None
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
stmt = select(TaskRun).order_by(desc(TaskRun.id))
|
||||||
|
if queue:
|
||||||
|
stmt = stmt.where(TaskRun.queue == queue)
|
||||||
|
if status:
|
||||||
|
stmt = stmt.where(TaskRun.status == status)
|
||||||
|
if before_id is not None:
|
||||||
|
stmt = stmt.where(TaskRun.id < before_id)
|
||||||
|
stmt = stmt.limit(limit + 1)
|
||||||
|
rows = (await session.execute(stmt)).scalars().all()
|
||||||
|
|
||||||
|
has_more = len(rows) > limit
|
||||||
|
rows = rows[:limit]
|
||||||
|
return jsonify({
|
||||||
|
"runs": [_row_to_dict(r) for r in rows],
|
||||||
|
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/failures", methods=["GET"])
|
||||||
|
async def list_failures():
|
||||||
|
"""Recent failures across all lanes (24h window).
|
||||||
|
|
||||||
|
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "50")), 200)
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
|
||||||
|
since = datetime.now(UTC) - timedelta(hours=24)
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
recent_stmt = (
|
||||||
|
select(TaskRun)
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||||
|
.where(TaskRun.finished_at >= since)
|
||||||
|
.order_by(desc(TaskRun.finished_at))
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
recent = (await session.execute(recent_stmt)).scalars().all()
|
||||||
|
|
||||||
|
count_stmt = (
|
||||||
|
select(TaskRun.error_type, func.count(TaskRun.id))
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||||
|
.where(TaskRun.finished_at >= since)
|
||||||
|
.group_by(TaskRun.error_type)
|
||||||
|
.order_by(desc(func.count(TaskRun.id)))
|
||||||
|
)
|
||||||
|
counts = (await session.execute(count_stmt)).all()
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
"recent": [_row_to_dict(r) for r in recent],
|
||||||
|
"count_by_type": {
|
||||||
|
(row[0] or "Unknown"): row[1]
|
||||||
|
for row in counts
|
||||||
|
},
|
||||||
|
"since": since.isoformat(),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _row_to_dict(r: TaskRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": r.id,
|
||||||
|
"queue": r.queue,
|
||||||
|
"task_name": r.task_name,
|
||||||
|
"target_id": r.target_id,
|
||||||
|
"celery_task_id": r.celery_task_id,
|
||||||
|
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||||
|
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||||
|
"duration_ms": r.duration_ms,
|
||||||
|
"status": r.status,
|
||||||
|
"error_type": r.error_type,
|
||||||
|
"error_message": r.error_message,
|
||||||
|
"retry_count": r.retry_count,
|
||||||
|
"worker_hostname": r.worker_hostname,
|
||||||
|
"args_summary": r.args_summary,
|
||||||
|
}
|
||||||
@@ -0,0 +1,264 @@
|
|||||||
|
"""FC-3h: /api/system/backup — create/list/restore/delete/tag for
|
||||||
|
DB + image backups.
|
||||||
|
|
||||||
|
Read endpoints are public on FC (operator-facing internal API; same
|
||||||
|
posture as /api/system/activity). Write endpoints take a typed
|
||||||
|
`confirm` body field that must match a server-generated token for
|
||||||
|
that backup row, to prevent click-to-destroy by stale browser tabs
|
||||||
|
or accidental cURL.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import desc, select
|
||||||
|
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import BackupRun, ImportSettings
|
||||||
|
|
||||||
|
system_backup_bp = Blueprint(
|
||||||
|
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||||
|
)
|
||||||
|
|
||||||
|
_KINDS = frozenset({"db", "images"})
|
||||||
|
_TAG_MAX_LEN = 64
|
||||||
|
_BACKUP_SETTINGS_FIELDS = (
|
||||||
|
"backup_db_nightly_enabled",
|
||||||
|
"backup_db_nightly_hour_utc",
|
||||||
|
"backup_db_keep_last_n",
|
||||||
|
"backup_images_keep_last_n",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
|
body = {"error": error}
|
||||||
|
body.update(extra)
|
||||||
|
return jsonify(body), status
|
||||||
|
|
||||||
|
|
||||||
|
def _row_to_dict(r: BackupRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": r.id,
|
||||||
|
"kind": r.kind,
|
||||||
|
"status": r.status,
|
||||||
|
"tag": r.tag,
|
||||||
|
"triggered_by": r.triggered_by,
|
||||||
|
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||||
|
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||||
|
"duration_seconds": (
|
||||||
|
int((r.finished_at - r.started_at).total_seconds())
|
||||||
|
if r.finished_at and r.started_at else None
|
||||||
|
),
|
||||||
|
"sql_path": r.sql_path,
|
||||||
|
"tar_path": r.tar_path,
|
||||||
|
"size_bytes": r.size_bytes,
|
||||||
|
"error": r.error,
|
||||||
|
"restored_from_id": r.restored_from_id,
|
||||||
|
"manifest": r.manifest or {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_tag(tag):
|
||||||
|
if tag is None:
|
||||||
|
return None
|
||||||
|
if not isinstance(tag, str):
|
||||||
|
return _bad("invalid_tag", detail="tag must be string or null")
|
||||||
|
tag = tag.strip()
|
||||||
|
if not tag:
|
||||||
|
return None
|
||||||
|
if len(tag) > _TAG_MAX_LEN:
|
||||||
|
return _bad("invalid_tag", detail=f"tag too long (max {_TAG_MAX_LEN})")
|
||||||
|
return tag
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_backup_settings_patch(body: dict):
|
||||||
|
if "backup_db_nightly_enabled" in body and not isinstance(
|
||||||
|
body["backup_db_nightly_enabled"], bool,
|
||||||
|
):
|
||||||
|
return _bad("invalid_value", detail="backup_db_nightly_enabled must be bool")
|
||||||
|
if "backup_db_nightly_hour_utc" in body:
|
||||||
|
v = body["backup_db_nightly_hour_utc"]
|
||||||
|
if not isinstance(v, int) or isinstance(v, bool) or not (0 <= v <= 23):
|
||||||
|
return _bad("invalid_value", detail="backup_db_nightly_hour_utc must be 0..23")
|
||||||
|
if "backup_db_keep_last_n" in body:
|
||||||
|
v = body["backup_db_keep_last_n"]
|
||||||
|
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
|
||||||
|
return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
|
||||||
|
if "backup_images_keep_last_n" in body:
|
||||||
|
v = body["backup_images_keep_last_n"]
|
||||||
|
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 100):
|
||||||
|
return _bad("invalid_value", detail="backup_images_keep_last_n must be 1..100")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/db", methods=["POST"])
|
||||||
|
async def trigger_db_backup():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
tag = _validate_tag(body.get("tag"))
|
||||||
|
if isinstance(tag, tuple):
|
||||||
|
return tag
|
||||||
|
from ..tasks.backup import backup_db_task
|
||||||
|
backup_db_task.delay(tag=tag, triggered_by="manual")
|
||||||
|
return jsonify({"status": "dispatched"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/images", methods=["POST"])
|
||||||
|
async def trigger_images_backup():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
tag = _validate_tag(body.get("tag"))
|
||||||
|
if isinstance(tag, tuple):
|
||||||
|
return tag
|
||||||
|
from ..tasks.backup import backup_images_task
|
||||||
|
backup_images_task.delay(tag=tag, triggered_by="manual")
|
||||||
|
return jsonify({"status": "dispatched"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/runs", methods=["GET"])
|
||||||
|
async def list_runs():
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "50")), 200)
|
||||||
|
except ValueError:
|
||||||
|
return _bad("invalid_limit")
|
||||||
|
if limit < 1:
|
||||||
|
return _bad("invalid_limit")
|
||||||
|
kind = request.args.get("kind")
|
||||||
|
if kind is not None and kind not in _KINDS:
|
||||||
|
return _bad("invalid_kind", detail=f"kind must be one of {sorted(_KINDS)}")
|
||||||
|
before_id_raw = request.args.get("before_id")
|
||||||
|
before_id = int(before_id_raw) if before_id_raw else None
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
stmt = select(BackupRun).order_by(desc(BackupRun.id))
|
||||||
|
if kind:
|
||||||
|
stmt = stmt.where(BackupRun.kind == kind)
|
||||||
|
if before_id is not None:
|
||||||
|
stmt = stmt.where(BackupRun.id < before_id)
|
||||||
|
stmt = stmt.limit(limit + 1)
|
||||||
|
rows = (await session.execute(stmt)).scalars().all()
|
||||||
|
|
||||||
|
has_more = len(rows) > limit
|
||||||
|
rows = rows[:limit]
|
||||||
|
return jsonify({
|
||||||
|
"runs": [_row_to_dict(r) for r in rows],
|
||||||
|
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/runs/<int:run_id>", methods=["GET"])
|
||||||
|
async def get_run(run_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
row = await session.get(BackupRun, run_id)
|
||||||
|
if row is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
return jsonify(_row_to_dict(row))
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/runs/<int:run_id>", methods=["PATCH"])
|
||||||
|
async def patch_run(run_id: int):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
if "tag" not in body:
|
||||||
|
return _bad("invalid_body", detail="tag required")
|
||||||
|
tag = _validate_tag(body["tag"])
|
||||||
|
if isinstance(tag, tuple):
|
||||||
|
return tag
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
row = await session.get(BackupRun, run_id)
|
||||||
|
if row is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
row.tag = tag
|
||||||
|
await session.commit()
|
||||||
|
await session.refresh(row)
|
||||||
|
return jsonify(_row_to_dict(row))
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/runs/<int:run_id>/restore", methods=["POST"])
|
||||||
|
async def trigger_restore(run_id: int):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
supplied = body.get("confirm", "")
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
row = await session.get(BackupRun, run_id)
|
||||||
|
if row is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
if row.status != "ok":
|
||||||
|
return _bad(
|
||||||
|
"not_restorable",
|
||||||
|
detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
|
||||||
|
)
|
||||||
|
expected = f"restore-{row.kind}-{row.id}"
|
||||||
|
if supplied != expected:
|
||||||
|
return _bad(
|
||||||
|
"confirm_mismatch",
|
||||||
|
detail=f"confirm must equal {expected!r}",
|
||||||
|
expected=expected,
|
||||||
|
)
|
||||||
|
kind = row.kind
|
||||||
|
|
||||||
|
if kind == "db":
|
||||||
|
from ..tasks.backup import restore_db_task
|
||||||
|
restore_db_task.delay(source_backup_run_id=run_id)
|
||||||
|
else: # 'images' (the only other value _KINDS allows via the trigger path)
|
||||||
|
from ..tasks.backup import restore_images_task
|
||||||
|
restore_images_task.delay(source_backup_run_id=run_id)
|
||||||
|
return jsonify({"status": "dispatched", "kind": kind}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
|
||||||
|
async def delete_run(run_id: int):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
supplied = body.get("confirm", "")
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
row = await session.get(BackupRun, run_id)
|
||||||
|
if row is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
expected = f"delete-{row.kind}-{row.id}"
|
||||||
|
if supplied != expected:
|
||||||
|
return _bad(
|
||||||
|
"confirm_mismatch",
|
||||||
|
detail=f"confirm must equal {expected!r}",
|
||||||
|
expected=expected,
|
||||||
|
)
|
||||||
|
from ..services import backup_service
|
||||||
|
backup_service.unlink_artifact_files(
|
||||||
|
sql_path=row.sql_path, tar_path=row.tar_path,
|
||||||
|
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||||
|
)
|
||||||
|
await session.delete(row)
|
||||||
|
await session.commit()
|
||||||
|
return "", 204
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/settings", methods=["GET"])
|
||||||
|
async def get_settings():
|
||||||
|
async with get_session() as session:
|
||||||
|
row = (await session.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
)).scalar_one()
|
||||||
|
return jsonify({
|
||||||
|
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
||||||
|
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
||||||
|
"backup_db_keep_last_n": row.backup_db_keep_last_n,
|
||||||
|
"backup_images_keep_last_n": row.backup_images_keep_last_n,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@system_backup_bp.route("/settings", methods=["PATCH"])
|
||||||
|
async def patch_settings():
|
||||||
|
body = await request.get_json(silent=True)
|
||||||
|
if not isinstance(body, dict):
|
||||||
|
return _bad("invalid_body", detail="body must be a JSON object")
|
||||||
|
|
||||||
|
err = _validate_backup_settings_patch(body)
|
||||||
|
if err is not None:
|
||||||
|
return err
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
row = (await session.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
)).scalar_one()
|
||||||
|
for field in _BACKUP_SETTINGS_FIELDS:
|
||||||
|
if field in body:
|
||||||
|
setattr(row, field, body[field])
|
||||||
|
await session.commit()
|
||||||
|
return await get_settings()
|
||||||
+32
-5
@@ -15,6 +15,7 @@ from ..services.tag_service import (
|
|||||||
TagService,
|
TagService,
|
||||||
TagValidationError,
|
TagValidationError,
|
||||||
)
|
)
|
||||||
|
from ..utils.tag_prefix import parse_kind_prefix
|
||||||
|
|
||||||
tags_bp = Blueprint("tags", __name__, url_prefix="/api")
|
tags_bp = Blueprint("tags", __name__, url_prefix="/api")
|
||||||
|
|
||||||
@@ -105,13 +106,39 @@ async def directory():
|
|||||||
|
|
||||||
@tags_bp.route("/tags", methods=["POST"])
|
@tags_bp.route("/tags", methods=["POST"])
|
||||||
async def create_tag():
|
async def create_tag():
|
||||||
|
"""Create a tag. Two input shapes accepted:
|
||||||
|
1. Explicit: {name, kind, fandom_id?} — caller already split, kind wins.
|
||||||
|
2. IR-suffix: {name} where name = "kind:Name" (e.g. "artist:Eric").
|
||||||
|
The server runs parse_kind_prefix(name) to derive kind; the colon
|
||||||
|
and prefix are stripped from the stored tag name. If no recognized
|
||||||
|
prefix is present, the kind defaults to `general`.
|
||||||
|
Explicit kind ALWAYS wins (backward-compat for existing callers).
|
||||||
|
"""
|
||||||
body = await request.get_json()
|
body = await request.get_json()
|
||||||
if not body or "name" not in body or "kind" not in body:
|
if not body or "name" not in body:
|
||||||
return jsonify({"error": "name and kind required"}), 400
|
return jsonify({"error": "name required"}), 400
|
||||||
name = body["name"]
|
name = body["name"]
|
||||||
kind = _coerce_kind(body["kind"])
|
explicit_kind_raw = body.get("kind")
|
||||||
if kind is None:
|
|
||||||
return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400
|
if explicit_kind_raw is not None:
|
||||||
|
# Caller provided kind — honor it; don't re-parse.
|
||||||
|
kind = _coerce_kind(explicit_kind_raw)
|
||||||
|
if kind is None:
|
||||||
|
return jsonify({"error": f"invalid kind {explicit_kind_raw!r}"}), 400
|
||||||
|
else:
|
||||||
|
# IR-style: parse "kind:Name" from the raw name.
|
||||||
|
parsed_kind, parsed_name = parse_kind_prefix(name)
|
||||||
|
if parsed_kind is not None:
|
||||||
|
name = parsed_name
|
||||||
|
kind = _coerce_kind(parsed_kind)
|
||||||
|
# parse_kind_prefix only returns kinds from KNOWN_KINDS which
|
||||||
|
# are all valid TagKind members, so _coerce_kind can't return
|
||||||
|
# None here — but defensive.
|
||||||
|
if kind is None:
|
||||||
|
return jsonify({"error": f"invalid kind {parsed_kind!r}"}), 400
|
||||||
|
else:
|
||||||
|
kind = TagKind.general
|
||||||
|
|
||||||
fandom_id = body.get("fandom_id")
|
fandom_id = body.get("fandom_id")
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
"""Thumbnail admin API: backfill trigger."""
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify
|
||||||
|
|
||||||
|
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||||
|
|
||||||
|
|
||||||
|
@thumbnails_bp.route("/backfill", methods=["POST"])
|
||||||
|
async def trigger_backfill():
|
||||||
|
from ..tasks.thumbnail import backfill_thumbnails
|
||||||
|
|
||||||
|
r = backfill_thumbnails.delay()
|
||||||
|
return jsonify({"celery_task_id": r.id}), 202
|
||||||
@@ -31,6 +31,9 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.migration",
|
"backend.app.tasks.migration",
|
||||||
"backend.app.tasks.ml",
|
"backend.app.tasks.ml",
|
||||||
"backend.app.tasks.download",
|
"backend.app.tasks.download",
|
||||||
|
"backend.app.tasks.backup",
|
||||||
|
"backend.app.tasks.admin",
|
||||||
|
"backend.app.tasks.library_audit",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
app.conf.update(
|
app.conf.update(
|
||||||
@@ -43,6 +46,9 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||||
|
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||||
|
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||||
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||||
},
|
},
|
||||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
@@ -81,9 +87,28 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
|
"recover-stalled-task-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
||||||
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
||||||
|
},
|
||||||
|
"prune-task-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.prune_task_runs",
|
||||||
|
"schedule": 86400.0, # daily
|
||||||
|
},
|
||||||
|
"fc3h-backup-db-nightly": {
|
||||||
|
"task": "backend.app.tasks.backup.backup_db_nightly",
|
||||||
|
"schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour
|
||||||
|
},
|
||||||
|
"fc3h-prune-backups": {
|
||||||
|
"task": "backend.app.tasks.backup.prune_backups",
|
||||||
|
"schedule": 86400.0, # daily
|
||||||
|
},
|
||||||
},
|
},
|
||||||
timezone="UTC",
|
timezone="UTC",
|
||||||
)
|
)
|
||||||
|
# FC-3i: register task_run signal handlers (side-effect import).
|
||||||
|
from . import celery_signals # noqa: F401
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""FC-3i: task_run lifecycle via Celery signals.
|
||||||
|
|
||||||
|
Subscribes to task_prerun / task_postrun / task_failure / task_retry
|
||||||
|
and persists one task_run row per task attempt. Drop-in for every
|
||||||
|
existing and future Celery task — no per-task instrumentation.
|
||||||
|
|
||||||
|
Signal handlers run inside the worker process (sync context); DB
|
||||||
|
writes go through the existing shared sync engine
|
||||||
|
(backend.app.tasks._sync_engine.sync_session_factory) — one engine
|
||||||
|
per worker process, not per-task, so we don't blow Postgres
|
||||||
|
max_connections under load (the reason FC-3g shared-engine fix
|
||||||
|
existed).
|
||||||
|
|
||||||
|
Failure-mode discipline (operator-pressed point): every handler is
|
||||||
|
wrapped in try/except that swallows + logs. If the DB is down or the
|
||||||
|
handler has a bug, the real task still runs — the dashboard goes
|
||||||
|
dark for that interval. Monitoring NEVER breaks the thing it's
|
||||||
|
monitoring.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
||||||
|
|
||||||
|
from .models import TaskRun
|
||||||
|
from .tasks._sync_engine import sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Celery-internal tasks that would generate dashboard noise without
|
||||||
|
# operational value. Conservative list; extend only when a specific
|
||||||
|
# task proves noisy.
|
||||||
|
_UNTRACKED_TASK_NAMES = frozenset({
|
||||||
|
"celery.chord_unlock",
|
||||||
|
"celery.backend_cleanup",
|
||||||
|
"celery.chunks",
|
||||||
|
})
|
||||||
|
|
||||||
|
_MAX_ERROR_MESSAGE_LEN = 2000
|
||||||
|
_MAX_ARGS_SUMMARY_LEN = 255
|
||||||
|
_MAX_WORKER_HOSTNAME_LEN = 128
|
||||||
|
|
||||||
|
# PostgreSQL Integer is signed 32-bit. Tasks called with a first-arg
|
||||||
|
# int outside this range (e.g. an absurdly large mock value, or a
|
||||||
|
# string-of-digits coercible to int but bigger than 2^31-1) would crash
|
||||||
|
# the INSERT with NumericValueOutOfRange. Bound the recorded value to
|
||||||
|
# the column's range; values outside become None (target_id is
|
||||||
|
# nullable, so this is safe).
|
||||||
|
_INT32_MAX = 2_147_483_647
|
||||||
|
_INT32_MIN = -2_147_483_648
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_for(task) -> str:
|
||||||
|
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||||
|
Keep in sync if task_routes is reordered."""
|
||||||
|
name = getattr(task, "name", "") or ""
|
||||||
|
if name.startswith("backend.app.tasks.import_file."):
|
||||||
|
return "import"
|
||||||
|
if name.startswith("backend.app.tasks.ml."):
|
||||||
|
return "ml"
|
||||||
|
if name.startswith("backend.app.tasks.thumbnail."):
|
||||||
|
return "thumbnail"
|
||||||
|
if name.startswith("backend.app.tasks.download."):
|
||||||
|
return "download"
|
||||||
|
if name.startswith("backend.app.tasks.scan."):
|
||||||
|
return "scan"
|
||||||
|
if name.startswith((
|
||||||
|
"backend.app.tasks.maintenance.",
|
||||||
|
"backend.app.tasks.migration.",
|
||||||
|
)):
|
||||||
|
return "maintenance"
|
||||||
|
return "default"
|
||||||
|
|
||||||
|
|
||||||
|
def _target_id_from_args(args) -> int | None:
|
||||||
|
"""Best-effort: if the first positional arg parses as int AND fits
|
||||||
|
in the column's signed-32-bit range, record it as target_id
|
||||||
|
(image_id, source_id, etc.). Never raises."""
|
||||||
|
if not args:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = int(args[0])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if value < _INT32_MIN or value > _INT32_MAX:
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _truncate(s, limit: int) -> str | None:
|
||||||
|
if s is None:
|
||||||
|
return None
|
||||||
|
text = str(s)
|
||||||
|
return text if len(text) <= limit else text[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_tracked(task_name: str | None) -> bool:
|
||||||
|
return bool(task_name) and task_name not in _UNTRACKED_TASK_NAMES
|
||||||
|
|
||||||
|
|
||||||
|
@task_prerun.connect
|
||||||
|
def _on_prerun(sender=None, task_id=None, task=None, args=None,
|
||||||
|
kwargs=None, **_):
|
||||||
|
if not _is_tracked(getattr(task, "name", None)):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
Session = sync_session_factory()
|
||||||
|
with Session() as session:
|
||||||
|
session.add(TaskRun(
|
||||||
|
celery_task_id=task_id or "",
|
||||||
|
queue=_queue_for(task),
|
||||||
|
task_name=task.name,
|
||||||
|
target_id=_target_id_from_args(args),
|
||||||
|
started_at=datetime.now(UTC),
|
||||||
|
status="running",
|
||||||
|
args_summary=_truncate(repr(args), _MAX_ARGS_SUMMARY_LEN),
|
||||||
|
worker_hostname=_truncate(
|
||||||
|
getattr(sender, "hostname", None),
|
||||||
|
_MAX_WORKER_HOSTNAME_LEN,
|
||||||
|
),
|
||||||
|
))
|
||||||
|
session.commit()
|
||||||
|
except Exception: # noqa: BLE001 — never break the worker
|
||||||
|
log.exception("task_run prerun insert failed (task=%s)",
|
||||||
|
getattr(task, "name", "?"))
|
||||||
|
|
||||||
|
|
||||||
|
def _finalize(task_id: str, *, status: str,
|
||||||
|
error_type: str | None = None,
|
||||||
|
error_message: str | None = None,
|
||||||
|
retry_count: int | None = None) -> None:
|
||||||
|
"""Shared write path for postrun/failure/retry. Picks the most-
|
||||||
|
recent task_run row for this celery_task_id that's still 'running'
|
||||||
|
(retries reuse the same celery_task_id; each new attempt's prerun
|
||||||
|
inserts a fresh row, so finalize targets the latest running row)."""
|
||||||
|
try:
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
Session = sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with Session() as session:
|
||||||
|
row = session.execute(
|
||||||
|
select(TaskRun)
|
||||||
|
.where(TaskRun.celery_task_id == task_id)
|
||||||
|
.where(TaskRun.status == "running")
|
||||||
|
.order_by(TaskRun.id.desc())
|
||||||
|
.limit(1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if row is None:
|
||||||
|
return # no prerun row (untracked, insert failed, or already finalized)
|
||||||
|
row.finished_at = now
|
||||||
|
row.duration_ms = int(
|
||||||
|
(now - row.started_at).total_seconds() * 1000
|
||||||
|
)
|
||||||
|
row.status = status
|
||||||
|
if error_type is not None:
|
||||||
|
row.error_type = _truncate(error_type, 128)
|
||||||
|
if error_message is not None:
|
||||||
|
row.error_message = _truncate(error_message, _MAX_ERROR_MESSAGE_LEN)
|
||||||
|
if retry_count is not None:
|
||||||
|
row.retry_count = retry_count
|
||||||
|
session.commit()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.exception("task_run finalize failed (task_id=%s)", task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@task_postrun.connect
|
||||||
|
def _on_postrun(sender=None, task_id=None, task=None, args=None,
|
||||||
|
kwargs=None, retval=None, state=None, **_):
|
||||||
|
if not _is_tracked(getattr(task, "name", None)):
|
||||||
|
return
|
||||||
|
# state is one of SUCCESS/FAILURE/RETRY/etc. Only handle SUCCESS;
|
||||||
|
# task_failure handles FAILURE explicitly (with the exception).
|
||||||
|
if state != "SUCCESS":
|
||||||
|
return
|
||||||
|
_finalize(task_id, status="ok")
|
||||||
|
|
||||||
|
|
||||||
|
@task_failure.connect
|
||||||
|
def _on_failure(sender=None, task_id=None, exception=None,
|
||||||
|
args=None, kwargs=None, einfo=None, **_):
|
||||||
|
if not _is_tracked(getattr(sender, "name", None)):
|
||||||
|
return
|
||||||
|
status = ("timeout"
|
||||||
|
if isinstance(exception, SoftTimeLimitExceeded)
|
||||||
|
else "error")
|
||||||
|
_finalize(
|
||||||
|
task_id, status=status,
|
||||||
|
error_type=type(exception).__name__ if exception else "Unknown",
|
||||||
|
error_message=str(exception) if exception else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@task_retry.connect
|
||||||
|
def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
||||||
|
if not _is_tracked(getattr(sender, "name", None)):
|
||||||
|
return
|
||||||
|
task_id = getattr(request, "id", None)
|
||||||
|
if not task_id:
|
||||||
|
return
|
||||||
|
# Mark current attempt's row as 'retry' (terminal for this row).
|
||||||
|
# The next attempt's task_prerun inserts a fresh row.
|
||||||
|
_finalize(
|
||||||
|
task_id, status="retry",
|
||||||
|
error_type=type(reason).__name__ if reason else "Retry",
|
||||||
|
error_message=str(reason) if reason else None,
|
||||||
|
retry_count=getattr(request, "retries", 0),
|
||||||
|
)
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from .app_setting import AppSetting
|
from .app_setting import AppSetting
|
||||||
from .artist import Artist
|
from .artist import Artist
|
||||||
|
from .backup_run import BackupRun
|
||||||
from .base import Base
|
from .base import Base
|
||||||
from .credential import Credential
|
from .credential import Credential
|
||||||
from .download_event import DownloadEvent
|
from .download_event import DownloadEvent
|
||||||
@@ -10,6 +11,7 @@ from .image_record import ImageRecord
|
|||||||
from .import_batch import ImportBatch
|
from .import_batch import ImportBatch
|
||||||
from .import_settings import ImportSettings
|
from .import_settings import ImportSettings
|
||||||
from .import_task import ImportTask
|
from .import_task import ImportTask
|
||||||
|
from .library_audit_run import LibraryAuditRun
|
||||||
from .migration_run import MigrationRun
|
from .migration_run import MigrationRun
|
||||||
from .ml_settings import MLSettings
|
from .ml_settings import MLSettings
|
||||||
from .post import Post
|
from .post import Post
|
||||||
@@ -21,11 +23,13 @@ from .tag_alias import TagAlias
|
|||||||
from .tag_allowlist import TagAllowlist
|
from .tag_allowlist import TagAllowlist
|
||||||
from .tag_reference_embedding import TagReferenceEmbedding
|
from .tag_reference_embedding import TagReferenceEmbedding
|
||||||
from .tag_suggestion_rejection import TagSuggestionRejection
|
from .tag_suggestion_rejection import TagSuggestionRejection
|
||||||
|
from .task_run import TaskRun
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
"AppSetting",
|
"AppSetting",
|
||||||
"Artist",
|
"Artist",
|
||||||
|
"BackupRun",
|
||||||
"Source",
|
"Source",
|
||||||
"Credential",
|
"Credential",
|
||||||
"Post",
|
"Post",
|
||||||
@@ -40,10 +44,12 @@ __all__ = [
|
|||||||
"ImportBatch",
|
"ImportBatch",
|
||||||
"ImportTask",
|
"ImportTask",
|
||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
|
"LibraryAuditRun",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
"MigrationRun",
|
"MigrationRun",
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
"TagAllowlist",
|
"TagAllowlist",
|
||||||
"TagReferenceEmbedding",
|
"TagReferenceEmbedding",
|
||||||
"TagSuggestionRejection",
|
"TagSuggestionRejection",
|
||||||
|
"TaskRun",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""FC-3h: backup_run — operator-facing artifact record for a backup run.
|
||||||
|
|
||||||
|
One row per backup attempt (kind='db' or 'images'). Lifecycle
|
||||||
|
tracking (started_at/finished_at/duration_ms/exception text) lives
|
||||||
|
in task_run from FC-3i — this row records artifact metadata: file
|
||||||
|
paths, sizes, tag (retention protection), and restore lineage via
|
||||||
|
restored_from_id.
|
||||||
|
|
||||||
|
Status values (String, not Postgres ENUM — per
|
||||||
|
feedback_check_existing_enums):
|
||||||
|
pending — created but task hasn't started yet (rare; usually
|
||||||
|
status starts as 'running' from the task body).
|
||||||
|
running — backup task is in flight.
|
||||||
|
ok — artifact successfully written.
|
||||||
|
error — task raised; error column populated.
|
||||||
|
restoring — this row represents a restore attempt (kind = restored
|
||||||
|
kind); linked to source via restored_from_id.
|
||||||
|
restored — restore completed successfully.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class BackupRun(Base):
|
||||||
|
__tablename__ = "backup_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="pending", index=True,
|
||||||
|
)
|
||||||
|
tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||||
|
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True,
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True,
|
||||||
|
)
|
||||||
|
sql_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
tar_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
manifest: Mapped[dict] = mapped_column(
|
||||||
|
JSON, nullable=False, default=dict, server_default="{}",
|
||||||
|
)
|
||||||
|
restored_from_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
@@ -1,14 +1,18 @@
|
|||||||
"""ImageProvenance — links an ImageRecord to a Post.
|
"""ImageProvenance — links an ImageRecord to a Post.
|
||||||
|
|
||||||
Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate
|
One image can have many provenance rows — different posts each contribute
|
||||||
rule (spec §3): when a downloaded image is a pHash dupe of an existing
|
metadata (enrich-on-duplicate rule, spec §3: a downloaded image that is a
|
||||||
record, we append a new provenance row to the existing record rather than
|
pHash dupe of an existing record gets a NEW provenance row for the new post
|
||||||
dropping the metadata.
|
appended, rather than the metadata being dropped). But the (image, post)
|
||||||
|
pair is unique — alembic 0021 enforces uq_image_provenance_image_post
|
||||||
|
after operator-flagged 2026-05-26 saw _apply_sidecar's existence-check +
|
||||||
|
INSERT race plant duplicates that then broke .scalar_one_or_none() on
|
||||||
|
every later deep-scan rederive (MultipleResultsFound).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func
|
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, UniqueConstraint, func
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -16,6 +20,12 @@ from .base import Base
|
|||||||
|
|
||||||
class ImageProvenance(Base):
|
class ImageProvenance(Base):
|
||||||
__tablename__ = "image_provenance"
|
__tablename__ = "image_provenance"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"image_record_id", "post_id",
|
||||||
|
name="uq_image_provenance_image_post",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
image_record_id: Mapped[int] = mapped_column(
|
image_record_id: Mapped[int] = mapped_column(
|
||||||
|
|||||||
@@ -26,6 +26,10 @@ class ImportBatch(Base):
|
|||||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||||
|
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||||
|
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||||
|
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
|
||||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||||
# running | complete | cancelled
|
# running | complete | cancelled
|
||||||
|
|||||||
@@ -49,3 +49,17 @@ class ImportSettings(Base):
|
|||||||
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=5
|
Integer, nullable=False, default=5
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# FC-3h backup knobs.
|
||||||
|
backup_db_nightly_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=False,
|
||||||
|
)
|
||||||
|
backup_db_nightly_hour_utc: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=3,
|
||||||
|
)
|
||||||
|
backup_db_keep_last_n: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=14,
|
||||||
|
)
|
||||||
|
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=3,
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""LibraryAuditRun — async transparency / single_color audit lifecycle.
|
||||||
|
|
||||||
|
State machine: running → ready → applied / cancelled / error.
|
||||||
|
matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run
|
||||||
|
reads it and routes through cleanup_service.delete_images.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryAuditRun(Base):
|
||||||
|
__tablename__ = "library_audit_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||||
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
)
|
||||||
|
# running | ready | applied | cancelled | error
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True,
|
||||||
|
)
|
||||||
|
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -35,8 +35,10 @@ class TagKind(StrEnum):
|
|||||||
series = "series"
|
series = "series"
|
||||||
archive = "archive"
|
archive = "archive"
|
||||||
post = "post"
|
post = "post"
|
||||||
meta = "meta"
|
# `meta` and `rating` retired by operator 2026-05-26 (alembic 0023).
|
||||||
rating = "rating"
|
# `artist` retired in FC-2d-vii-c — artists are first-class entities
|
||||||
|
# via Artist/Source rows now, not tags — but the enum value stays
|
||||||
|
# to keep historic tag rows queryable.
|
||||||
|
|
||||||
|
|
||||||
image_tag = Table(
|
image_tag = Table(
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""FC-3i: task_run — per-Celery-task lifecycle audit row.
|
||||||
|
|
||||||
|
One row inserted by the task_prerun signal at task start, updated by
|
||||||
|
task_postrun / task_failure / task_retry. The shape supports the
|
||||||
|
SystemActivity dashboard's three panes: per-queue queue+worker summary,
|
||||||
|
recent failures (24h, grouped by error_type), and full paginated
|
||||||
|
activity history.
|
||||||
|
|
||||||
|
Retention: ok rows pruned after 24h, error/timeout after 7d (see
|
||||||
|
backend.app.tasks.maintenance.prune_task_runs).
|
||||||
|
|
||||||
|
Recovery: rows stuck in 'running' for >5 min flipped to 'error' by
|
||||||
|
backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Integer, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class TaskRun(Base):
|
||||||
|
__tablename__ = "task_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
celery_task_id: Mapped[str] = mapped_column(
|
||||||
|
String(64), nullable=False, index=True,
|
||||||
|
)
|
||||||
|
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||||
|
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||||
|
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True,
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True,
|
||||||
|
)
|
||||||
|
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
)
|
||||||
|
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
retry_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
worker_hostname: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
args_summary: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
@@ -25,12 +25,31 @@ def _snapshot(repo_id: str, dest: Path, allow_patterns: list[str] | None) -> Non
|
|||||||
|
|
||||||
|
|
||||||
def ensure_camie() -> None:
|
def ensure_camie() -> None:
|
||||||
|
"""Fetch Camie v2 weights + metadata.
|
||||||
|
|
||||||
|
v2 layout (HuggingFace Camais03/camie-tagger-v2): the ONNX file is
|
||||||
|
named camie-tagger-v2.onnx (not model.onnx) and tags ship inside
|
||||||
|
camie-tagger-v2-metadata.json (not selected_tags.csv). Both at root.
|
||||||
|
The repo also contains app/, game/, training/, images/ subdirs full
|
||||||
|
of setup/demo files we don't need — allow_patterns scopes the fetch
|
||||||
|
to just the inference essentials (~790 MB instead of ~2 GB).
|
||||||
|
"""
|
||||||
dest = MODEL_ROOT / "camie"
|
dest = MODEL_ROOT / "camie"
|
||||||
if (dest / "model.onnx").is_file() and (dest / "selected_tags.csv").is_file():
|
model_file = dest / "camie-tagger-v2.onnx"
|
||||||
|
meta_file = dest / "camie-tagger-v2-metadata.json"
|
||||||
|
if model_file.is_file() and meta_file.is_file():
|
||||||
print(f"[download_models] Camie present at {dest}")
|
print(f"[download_models] Camie present at {dest}")
|
||||||
return
|
return
|
||||||
print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}")
|
print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}")
|
||||||
_snapshot(CAMIE_REPO, dest, ["model.onnx", "selected_tags.csv", "*.json"])
|
_snapshot(
|
||||||
|
CAMIE_REPO, dest,
|
||||||
|
[
|
||||||
|
"camie-tagger-v2.onnx",
|
||||||
|
"camie-tagger-v2-metadata.json",
|
||||||
|
"config.json",
|
||||||
|
"config.yaml",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def ensure_siglip() -> None:
|
def ensure_siglip() -> None:
|
||||||
|
|||||||
@@ -111,12 +111,22 @@ class ArtistService:
|
|||||||
)
|
)
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
|
post_count = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(func.count(func.distinct(Post.id)))
|
||||||
|
.select_from(Post)
|
||||||
|
.join(Source, Source.id == Post.source_id)
|
||||||
|
.where(Source.artist_id == aid)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"id": artist.id,
|
"id": artist.id,
|
||||||
"name": artist.name,
|
"name": artist.name,
|
||||||
"slug": artist.slug,
|
"slug": artist.slug,
|
||||||
"is_subscription": bool(artist.is_subscription),
|
"is_subscription": bool(artist.is_subscription),
|
||||||
"image_count": int(image_count),
|
"image_count": int(image_count),
|
||||||
|
"post_count": int(post_count),
|
||||||
"date_range": {
|
"date_range": {
|
||||||
"min": dmin.isoformat() if dmin else None,
|
"min": dmin.isoformat() if dmin else None,
|
||||||
"max": dmax.isoformat() if dmax else None,
|
"max": dmax.isoformat() if dmax else None,
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool.
|
||||||
|
|
||||||
|
The retroactive library-cleanup tab and (future) import-time filter logic
|
||||||
|
both consume these. Importers should NOT inline rule logic going forward;
|
||||||
|
add the rule here and call from both sides.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""Single-color audit: matches images where one color dominates beyond
|
||||||
|
the threshold (within the given Euclidean RGB tolerance). The first
|
||||||
|
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||||
|
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||||
|
future spec can adopt it on the import path too.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
_THUMB_SIZE = (64, 64)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(
|
||||||
|
pil_image,
|
||||||
|
*,
|
||||||
|
threshold: float,
|
||||||
|
tolerance: int,
|
||||||
|
) -> bool:
|
||||||
|
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||||
|
distance) of the dominant color exceeds `threshold`.
|
||||||
|
|
||||||
|
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||||
|
Alpha channels are stripped; only RGB is considered. Animated images
|
||||||
|
use frame 0 (PIL's default after Image.open without seek).
|
||||||
|
"""
|
||||||
|
im = pil_image
|
||||||
|
if im.mode == "RGBA":
|
||||||
|
im = im.convert("RGB")
|
||||||
|
elif im.mode not in ("RGB", "L"):
|
||||||
|
im = im.convert("RGB")
|
||||||
|
if im.size != _THUMB_SIZE:
|
||||||
|
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||||
|
pixels = list(im.getdata())
|
||||||
|
if not pixels:
|
||||||
|
return False
|
||||||
|
# Normalize L-mode pixels to RGB tuples for distance math.
|
||||||
|
if isinstance(pixels[0], int):
|
||||||
|
pixels = [(p, p, p) for p in pixels]
|
||||||
|
# Dominant color = mean RGB.
|
||||||
|
n = len(pixels)
|
||||||
|
sum_r = sum(p[0] for p in pixels)
|
||||||
|
sum_g = sum(p[1] for p in pixels)
|
||||||
|
sum_b = sum(p[2] for p in pixels)
|
||||||
|
dom = (sum_r / n, sum_g / n, sum_b / n)
|
||||||
|
tol_sq = tolerance * tolerance
|
||||||
|
within = 0
|
||||||
|
for r, g, b in pixels:
|
||||||
|
dr = r - dom[0]
|
||||||
|
dg = g - dom[1]
|
||||||
|
db = b - dom[2]
|
||||||
|
if dr * dr + dg * dg + db * db <= tol_sq:
|
||||||
|
within += 1
|
||||||
|
return (within / n) > threshold
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""Transparency audit: matches images whose transparent-pixel fraction
|
||||||
|
exceeds the threshold. Animated images short-circuit (skipped) to avoid
|
||||||
|
the multi-frame PIL decode that hits Celery's hard time limit."""
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(pil_image, *, threshold: float) -> bool:
|
||||||
|
"""True iff the image's transparent-pixel fraction exceeds threshold.
|
||||||
|
|
||||||
|
False for non-alpha modes and animated images. Mirrors the import-side
|
||||||
|
Importer._transparency_pct logic so retroactive enforcement matches
|
||||||
|
prospective filtering.
|
||||||
|
"""
|
||||||
|
if getattr(pil_image, "is_animated", False):
|
||||||
|
return False
|
||||||
|
if pil_image.mode not in ("RGBA", "LA") and not (
|
||||||
|
pil_image.mode == "P" and "transparency" in pil_image.info
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
im = pil_image
|
||||||
|
if im.mode != "RGBA":
|
||||||
|
im = im.convert("RGBA")
|
||||||
|
alpha = im.getchannel("A")
|
||||||
|
histogram = alpha.histogram()
|
||||||
|
transparent = histogram[0]
|
||||||
|
total = sum(histogram)
|
||||||
|
pct = transparent / total if total else 0.0
|
||||||
|
return pct > threshold
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
"""FC-3h: first-class backup/restore service for FC.
|
||||||
|
|
||||||
|
Two independent backup kinds:
|
||||||
|
- 'db' — pg_dump only; fast; nightly via Beat (settings-gated)
|
||||||
|
- 'images' — tar+zstd of /images; slow; manual trigger only
|
||||||
|
|
||||||
|
Files live under <images_root>/_backups/. Each backup writes:
|
||||||
|
fc_<kind>_<ts>.{sql|tar.zst} — the artifact
|
||||||
|
fc_<kind>_<ts>.json — manifest (kind/tag/triggered_by)
|
||||||
|
|
||||||
|
Service functions are sync (subprocess-bound). Celery tasks in
|
||||||
|
backend.app.tasks.backup wrap each one with task_run-tracked
|
||||||
|
lifecycle + soft/hard time limits + retention bookkeeping.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_BACKUPS_DIRNAME = "_backups"
|
||||||
|
|
||||||
|
# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The
|
||||||
|
# Celery soft limit signals the Python process; subprocess.Popen in a
|
||||||
|
# blocking syscall ignores that signal. These bound the worst case.
|
||||||
|
_DB_SUBPROCESS_TIMEOUT_S = 12 * 60 # 12 min (Celery soft is 10 min)
|
||||||
|
_IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60 # 7 hr (Celery soft is 6 hr)
|
||||||
|
|
||||||
|
|
||||||
|
def _libpq_url(sa_url: str) -> str:
|
||||||
|
"""Strip SQLAlchemy +psycopg/+asyncpg driver suffix for pg_dump/psql."""
|
||||||
|
for driver in (
|
||||||
|
"postgresql+psycopg",
|
||||||
|
"postgresql+asyncpg",
|
||||||
|
"postgresql+psycopg2",
|
||||||
|
):
|
||||||
|
if sa_url.startswith(driver + "://"):
|
||||||
|
return "postgresql://" + sa_url[len(driver) + 3:]
|
||||||
|
return sa_url
|
||||||
|
|
||||||
|
|
||||||
|
def _backups_dir(images_root: Path) -> Path:
|
||||||
|
p = images_root / _BACKUPS_DIRNAME
|
||||||
|
p.mkdir(parents=True, exist_ok=True)
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def _now_ts() -> str:
|
||||||
|
return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def _file_size_or_none(path: Path) -> int | None:
|
||||||
|
try:
|
||||||
|
return path.stat().st_size
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _write_manifest(
|
||||||
|
out_dir: Path, *, kind: str, ts: str,
|
||||||
|
tag: str | None, triggered_by: str,
|
||||||
|
artifact_path: Path,
|
||||||
|
) -> Path:
|
||||||
|
manifest = {
|
||||||
|
"kind": kind,
|
||||||
|
"backup_id": f"fc_{kind}_{ts}",
|
||||||
|
"tag": tag,
|
||||||
|
"triggered_by": triggered_by,
|
||||||
|
"created_at": datetime.now(UTC).isoformat(),
|
||||||
|
"artifact_path": str(artifact_path),
|
||||||
|
}
|
||||||
|
mf = out_dir / f"fc_{kind}_{ts}.json"
|
||||||
|
mf.write_text(json.dumps(manifest, indent=2))
|
||||||
|
return mf
|
||||||
|
|
||||||
|
|
||||||
|
def backup_db(
|
||||||
|
*, db_url: str, images_root: Path,
|
||||||
|
tag: str | None = None, triggered_by: str = "manual",
|
||||||
|
) -> dict:
|
||||||
|
"""Run pg_dump; write .sql + manifest; return dict for the caller
|
||||||
|
to persist into BackupRun. Raises on subprocess failure."""
|
||||||
|
ts = _now_ts()
|
||||||
|
out_dir = _backups_dir(images_root)
|
||||||
|
sql_path = out_dir / f"fc_db_{ts}.sql"
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"pg_dump", "--no-owner", "--no-acl",
|
||||||
|
"-f", str(sql_path), _libpq_url(db_url),
|
||||||
|
],
|
||||||
|
capture_output=True, check=True,
|
||||||
|
timeout=_DB_SUBPROCESS_TIMEOUT_S,
|
||||||
|
)
|
||||||
|
manifest_path = _write_manifest(
|
||||||
|
out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||||
|
artifact_path=sql_path,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"kind": "db",
|
||||||
|
"ts": ts,
|
||||||
|
"sql_path": str(sql_path),
|
||||||
|
"tar_path": None,
|
||||||
|
"manifest_path": str(manifest_path),
|
||||||
|
"size_bytes": _file_size_or_none(sql_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def backup_images(
|
||||||
|
*, images_root: Path,
|
||||||
|
tag: str | None = None, triggered_by: str = "manual",
|
||||||
|
) -> dict:
|
||||||
|
"""Run tar --zstd over images_root; write .tar.zst + manifest."""
|
||||||
|
ts = _now_ts()
|
||||||
|
out_dir = _backups_dir(images_root)
|
||||||
|
tar_path = out_dir / f"fc_images_{ts}.tar.zst"
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"tar", "--zstd", "-cf", str(tar_path),
|
||||||
|
"-C", str(images_root.parent), images_root.name,
|
||||||
|
f"--exclude={images_root.name}/_backups",
|
||||||
|
f"--exclude={images_root.name}/_quarantine",
|
||||||
|
],
|
||||||
|
capture_output=True, check=True,
|
||||||
|
timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||||
|
)
|
||||||
|
manifest_path = _write_manifest(
|
||||||
|
out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||||
|
artifact_path=tar_path,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"kind": "images",
|
||||||
|
"ts": ts,
|
||||||
|
"sql_path": None,
|
||||||
|
"tar_path": str(tar_path),
|
||||||
|
"manifest_path": str(manifest_path),
|
||||||
|
"size_bytes": _file_size_or_none(tar_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def restore_db(*, db_url: str, sql_path: Path) -> None:
|
||||||
|
"""Wipe public schema, then load from .sql. Raises on subprocess
|
||||||
|
failure; partial-restore state is the caller's concern."""
|
||||||
|
libpq = _libpq_url(db_url)
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"psql", libpq, "-c",
|
||||||
|
"DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
|
||||||
|
],
|
||||||
|
capture_output=True, check=True, timeout=120,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["psql", libpq, "-f", str(sql_path)],
|
||||||
|
capture_output=True, check=True,
|
||||||
|
timeout=_DB_SUBPROCESS_TIMEOUT_S,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def restore_images(*, images_root: Path, tar_path: Path) -> None:
|
||||||
|
"""Untar over images_root.parent. Additive — files NOT in the
|
||||||
|
tarball are NOT removed. Caller wipes first if a clean restore
|
||||||
|
is needed."""
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"tar", "--zstd", "-xf", str(tar_path),
|
||||||
|
"-C", str(images_root.parent),
|
||||||
|
],
|
||||||
|
capture_output=True, check=True,
|
||||||
|
timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def unlink_artifact_files(
|
||||||
|
*,
|
||||||
|
sql_path: str | None,
|
||||||
|
tar_path: str | None,
|
||||||
|
manifest_path: str | None,
|
||||||
|
) -> dict:
|
||||||
|
"""Best-effort unlink of all on-disk files for a BackupRun row.
|
||||||
|
Returns dict keyed by label with True/False per file. Missing
|
||||||
|
files count as success (missing_ok semantics)."""
|
||||||
|
deleted: dict = {}
|
||||||
|
for label, p in (
|
||||||
|
("sql", sql_path),
|
||||||
|
("tar", tar_path),
|
||||||
|
("manifest", manifest_path),
|
||||||
|
):
|
||||||
|
if not p:
|
||||||
|
continue
|
||||||
|
path = Path(p)
|
||||||
|
try:
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
deleted[label] = True
|
||||||
|
except OSError:
|
||||||
|
deleted[label] = False
|
||||||
|
return deleted
|
||||||
@@ -0,0 +1,512 @@
|
|||||||
|
"""FC-3k: first-class admin destructive operations.
|
||||||
|
|
||||||
|
Projections are pure SELECTs used by both dry-run preview endpoints
|
||||||
|
and Tier-B count prompts. Mutations (Task 2) are called from sync
|
||||||
|
HTTP handlers (small ops) and from Celery tasks in
|
||||||
|
backend.app.tasks.admin (long ops).
|
||||||
|
|
||||||
|
This module is the PERMANENT home of artist-cascade + image-unlink
|
||||||
|
logic. The legacy copy at backend/app/services/migrators/cleanup.py
|
||||||
|
stays in place until FC-3j; FC-3j will replace its body with thin
|
||||||
|
re-exports from this module and then delete the wrapper.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, update
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||||
|
from ..models.series_page import SeriesPage
|
||||||
|
from ..models.tag import image_tag
|
||||||
|
|
||||||
|
|
||||||
|
def project_artist_cascade(session: Session, *, slug: str) -> dict:
|
||||||
|
"""Read-only projection of what delete_artist_cascade would touch.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"artist": {"id": int, "name": str, "slug": str},
|
||||||
|
"projected": {
|
||||||
|
"images": int,
|
||||||
|
"sources": int,
|
||||||
|
"thumbs": int, # images with a thumbnail_path set
|
||||||
|
"import_tasks": int, # ImportTask rows referencing the artist's images
|
||||||
|
"bytes_on_disk": int, # SUM(image_record.size_bytes) — column is NOT NULL
|
||||||
|
},
|
||||||
|
}
|
||||||
|
Raises LookupError if slug not found. No mutations.
|
||||||
|
"""
|
||||||
|
from ..models.import_task import ImportTask
|
||||||
|
from ..models.source import Source
|
||||||
|
|
||||||
|
artist = session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slug)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
raise LookupError(f"artist slug not found: {slug!r}")
|
||||||
|
|
||||||
|
images_count = session.execute(
|
||||||
|
select(func.count(ImageRecord.id))
|
||||||
|
.where(ImageRecord.artist_id == artist.id)
|
||||||
|
).scalar_one()
|
||||||
|
sources_count = session.execute(
|
||||||
|
select(func.count(Source.id))
|
||||||
|
.where(Source.artist_id == artist.id)
|
||||||
|
).scalar_one()
|
||||||
|
thumbs_count = session.execute(
|
||||||
|
select(func.count(ImageRecord.id))
|
||||||
|
.where(ImageRecord.artist_id == artist.id)
|
||||||
|
.where(ImageRecord.thumbnail_path.is_not(None))
|
||||||
|
).scalar_one()
|
||||||
|
import_tasks_count = session.execute(
|
||||||
|
select(func.count(ImportTask.id))
|
||||||
|
.where(
|
||||||
|
ImportTask.result_image_id.in_(
|
||||||
|
select(ImageRecord.id).where(ImageRecord.artist_id == artist.id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
bytes_on_disk = session.execute(
|
||||||
|
select(func.coalesce(func.sum(ImageRecord.size_bytes), 0))
|
||||||
|
.where(ImageRecord.artist_id == artist.id)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
|
"projected": {
|
||||||
|
"images": images_count,
|
||||||
|
"sources": sources_count,
|
||||||
|
"thumbs": thumbs_count,
|
||||||
|
"import_tasks": import_tasks_count,
|
||||||
|
"bytes_on_disk": int(bytes_on_disk),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def project_bulk_image_delete(
|
||||||
|
session: Session, *, image_ids: list[int],
|
||||||
|
) -> dict:
|
||||||
|
"""Read-only projection of what delete_images would touch.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"images_found": int,
|
||||||
|
"thumbs_to_unlink": int,
|
||||||
|
"bytes_on_disk": int,
|
||||||
|
"missing_ids": list[int], # ids passed in that don't exist
|
||||||
|
}
|
||||||
|
No mutations.
|
||||||
|
"""
|
||||||
|
if not image_ids:
|
||||||
|
return {
|
||||||
|
"images_found": 0,
|
||||||
|
"thumbs_to_unlink": 0,
|
||||||
|
"bytes_on_disk": 0,
|
||||||
|
"missing_ids": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
ImageRecord.id,
|
||||||
|
ImageRecord.thumbnail_path,
|
||||||
|
ImageRecord.size_bytes,
|
||||||
|
).where(ImageRecord.id.in_(image_ids))
|
||||||
|
).all()
|
||||||
|
found_ids = {r.id for r in rows}
|
||||||
|
missing = sorted(set(image_ids) - found_ids)
|
||||||
|
return {
|
||||||
|
"images_found": len(rows),
|
||||||
|
"thumbs_to_unlink": sum(1 for r in rows if r.thumbnail_path),
|
||||||
|
"bytes_on_disk": sum(r.size_bytes for r in rows),
|
||||||
|
"missing_ids": missing,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def count_tag_associations(session: Session, *, tag_id: int) -> int:
|
||||||
|
"""COUNT(*) FROM image_tag WHERE tag_id=?. For Tier-B prompt."""
|
||||||
|
return session.execute(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(image_tag)
|
||||||
|
.where(image_tag.c.tag_id == tag_id)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
def find_unused_tags(
|
||||||
|
session: Session, *, limit: int | None = None,
|
||||||
|
) -> list[Tag]:
|
||||||
|
"""Tags with no image_tag rows AND no series_page rows.
|
||||||
|
|
||||||
|
Sorted by name. Used by both dry-run preview and the live prune.
|
||||||
|
A tag is "unused" iff it has zero rows in image_tag AND zero rows
|
||||||
|
in series_page (so we don't accidentally prune a series tag that
|
||||||
|
happens to have no images yet).
|
||||||
|
"""
|
||||||
|
used_via_image_tag = select(image_tag.c.tag_id).distinct()
|
||||||
|
used_via_series = select(SeriesPage.series_tag_id).where(
|
||||||
|
SeriesPage.series_tag_id.is_not(None)
|
||||||
|
).distinct()
|
||||||
|
stmt = (
|
||||||
|
select(Tag)
|
||||||
|
.where(Tag.id.not_in(used_via_image_tag))
|
||||||
|
.where(Tag.id.not_in(used_via_series))
|
||||||
|
.order_by(Tag.name)
|
||||||
|
)
|
||||||
|
if limit is not None:
|
||||||
|
stmt = stmt.limit(limit)
|
||||||
|
return list(session.execute(stmt).scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
def unlink_image_files(
|
||||||
|
image: ImageRecord, images_root: Path,
|
||||||
|
) -> dict:
|
||||||
|
"""Best-effort unlink of all on-disk files for an ImageRecord.
|
||||||
|
|
||||||
|
Targets: image.path (original), image.thumbnail_path (cached
|
||||||
|
thumbnail), and the computed thumbs path at
|
||||||
|
/images/thumbs/<sha256[:3]>/<sha256>.(jpg|png|webp) (tries all
|
||||||
|
three extensions; missing extension is silently OK).
|
||||||
|
|
||||||
|
Returns {"original": bool, "thumbnail": bool}. Missing files
|
||||||
|
count as success (missing_ok semantics). OSErrors are swallowed
|
||||||
|
and reported as False so the calling DB delete still proceeds.
|
||||||
|
"""
|
||||||
|
out = {"original": False, "thumbnail": False}
|
||||||
|
if image.path:
|
||||||
|
try:
|
||||||
|
Path(image.path).unlink(missing_ok=True)
|
||||||
|
out["original"] = True
|
||||||
|
except OSError:
|
||||||
|
out["original"] = False
|
||||||
|
# Custom thumbnail_path (when set) — try it first.
|
||||||
|
if image.thumbnail_path:
|
||||||
|
try:
|
||||||
|
Path(image.thumbnail_path).unlink(missing_ok=True)
|
||||||
|
out["thumbnail"] = True
|
||||||
|
except OSError:
|
||||||
|
out["thumbnail"] = False
|
||||||
|
# Convention thumbs dir — try all extensions; missing OK.
|
||||||
|
if image.sha256:
|
||||||
|
bucket = image.sha256[:3]
|
||||||
|
for ext in ("jpg", "png", "webp"):
|
||||||
|
try:
|
||||||
|
(images_root / "thumbs" / bucket / f"{image.sha256}.{ext}").unlink(
|
||||||
|
missing_ok=True,
|
||||||
|
)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def delete_artist_cascade(
|
||||||
|
session: Session, *, artist_id: int, images_root: Path,
|
||||||
|
) -> dict:
|
||||||
|
"""Batched delete of an artist's images + the artist row.
|
||||||
|
|
||||||
|
Mirrors the cleanup_artist_async pattern: 500-row batches,
|
||||||
|
commit between batches so partial progress survives a worker
|
||||||
|
kill. Idempotent on missing artist (returns zeroed counts).
|
||||||
|
Postgres cascades handle image_tag / image_provenance /
|
||||||
|
series_page / tag_suggestion_rejection from ImageRecord delete,
|
||||||
|
and source / post / download_event / etc. from Artist delete
|
||||||
|
(via Artist.sources cascade="all, delete-orphan").
|
||||||
|
"""
|
||||||
|
artist = session.get(Artist, artist_id)
|
||||||
|
if artist is None:
|
||||||
|
return {
|
||||||
|
"artist": None,
|
||||||
|
"summary": {
|
||||||
|
"images_deleted": 0,
|
||||||
|
"files_deleted": 0,
|
||||||
|
"thumbs_deleted": 0,
|
||||||
|
"import_tasks_nulled": 0,
|
||||||
|
"files_failed": 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
artist_info = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||||
|
|
||||||
|
images_deleted = 0
|
||||||
|
files_deleted = 0
|
||||||
|
thumbs_deleted = 0
|
||||||
|
files_failed = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord)
|
||||||
|
.where(ImageRecord.artist_id == artist.id)
|
||||||
|
.limit(500)
|
||||||
|
).scalars().all()
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for img in rows:
|
||||||
|
unlinked = unlink_image_files(img, images_root)
|
||||||
|
if unlinked["original"]:
|
||||||
|
files_deleted += 1
|
||||||
|
else:
|
||||||
|
files_failed += 1
|
||||||
|
if unlinked["thumbnail"]:
|
||||||
|
thumbs_deleted += 1
|
||||||
|
session.delete(img)
|
||||||
|
images_deleted += 1
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
# ImportTask.result_image_id FK is SET NULL on image delete (Postgres
|
||||||
|
# handles this in the cascade above). We don't separately count those
|
||||||
|
# in FC-3k — the legacy cleanup_artist_async did it via
|
||||||
|
# source_path_prefix matching that's out of scope here.
|
||||||
|
import_tasks_nulled = 0
|
||||||
|
|
||||||
|
session.delete(artist)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"artist": artist_info,
|
||||||
|
"summary": {
|
||||||
|
"images_deleted": images_deleted,
|
||||||
|
"files_deleted": files_deleted,
|
||||||
|
"thumbs_deleted": thumbs_deleted,
|
||||||
|
"import_tasks_nulled": import_tasks_nulled,
|
||||||
|
"files_failed": files_failed,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def delete_images(
|
||||||
|
session: Session, *, image_ids: list[int], images_root: Path,
|
||||||
|
) -> dict:
|
||||||
|
"""Delete a list of images in 500-row batches with commit between.
|
||||||
|
|
||||||
|
Postgres CASCADE on image_tag / image_provenance / series_page /
|
||||||
|
tag_suggestion_rejection / post_attachment(FK SET NULL) handles
|
||||||
|
the DB side; this function handles file unlinks first then row
|
||||||
|
deletes. Idempotent on missing IDs (returned as missing_ids;
|
||||||
|
no error). On partial OSError, the row is still deleted and
|
||||||
|
files_failed is incremented.
|
||||||
|
"""
|
||||||
|
if not image_ids:
|
||||||
|
return {
|
||||||
|
"images_deleted": 0,
|
||||||
|
"files_deleted": 0,
|
||||||
|
"thumbs_deleted": 0,
|
||||||
|
"files_failed": 0,
|
||||||
|
"missing_ids": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
seen_ids: set[int] = set()
|
||||||
|
images_deleted = 0
|
||||||
|
files_deleted = 0
|
||||||
|
thumbs_deleted = 0
|
||||||
|
files_failed = 0
|
||||||
|
|
||||||
|
pending = list(image_ids)
|
||||||
|
while pending:
|
||||||
|
batch_ids = pending[:500]
|
||||||
|
pending = pending[500:]
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord).where(ImageRecord.id.in_(batch_ids))
|
||||||
|
).scalars().all()
|
||||||
|
for img in rows:
|
||||||
|
seen_ids.add(img.id)
|
||||||
|
unlinked = unlink_image_files(img, images_root)
|
||||||
|
if unlinked["original"]:
|
||||||
|
files_deleted += 1
|
||||||
|
else:
|
||||||
|
files_failed += 1
|
||||||
|
if unlinked["thumbnail"]:
|
||||||
|
thumbs_deleted += 1
|
||||||
|
session.delete(img)
|
||||||
|
images_deleted += 1
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
missing = sorted(set(image_ids) - seen_ids)
|
||||||
|
return {
|
||||||
|
"images_deleted": images_deleted,
|
||||||
|
"files_deleted": files_deleted,
|
||||||
|
"thumbs_deleted": thumbs_deleted,
|
||||||
|
"files_failed": files_failed,
|
||||||
|
"missing_ids": missing,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def delete_tag(session: Session, *, tag_id: int) -> dict:
|
||||||
|
"""Simple DELETE FROM tag WHERE id=?.
|
||||||
|
|
||||||
|
Postgres cascades the rest (image_tag, tag_alias, tag_allowlist,
|
||||||
|
tag_reference_embedding, tag_suggestion_rejection, series_page).
|
||||||
|
Returns counts BEFORE delete so the caller can surface them.
|
||||||
|
Raises LookupError if tag_id not found.
|
||||||
|
"""
|
||||||
|
tag = session.get(Tag, tag_id)
|
||||||
|
if tag is None:
|
||||||
|
raise LookupError(f"tag id not found: {tag_id}")
|
||||||
|
associations_count = count_tag_associations(session, tag_id=tag_id)
|
||||||
|
info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
|
||||||
|
session.delete(tag)
|
||||||
|
session.commit()
|
||||||
|
return {"deleted": info, "associations_removed": associations_count}
|
||||||
|
|
||||||
|
|
||||||
|
def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||||
|
"""Find tags with zero references and (unless dry_run) delete them.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dry_run=True: {"count": N, "sample_names": [first 50]}
|
||||||
|
dry_run=False: {"deleted": N, "sample_names": [first 50]}
|
||||||
|
"""
|
||||||
|
unused = find_unused_tags(session)
|
||||||
|
sample = [t.name for t in unused[:50]]
|
||||||
|
if dry_run:
|
||||||
|
return {"count": len(unused), "sample_names": sample}
|
||||||
|
ids = [t.id for t in unused]
|
||||||
|
if ids:
|
||||||
|
session.execute(
|
||||||
|
Tag.__table__.delete().where(Tag.id.in_(ids))
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {"deleted": len(ids), "sample_names": sample}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_MIN_DIM_SAMPLE_CAP = 50
|
||||||
|
|
||||||
|
|
||||||
|
def project_min_dimension_violations(
|
||||||
|
session: Session, *, min_width: int, min_height: int,
|
||||||
|
) -> dict:
|
||||||
|
"""Return {count, sample_ids} for image_record rows with width or
|
||||||
|
height below the thresholds. Synchronous SQL — no PIL inspection
|
||||||
|
needed since width/height are stored columns."""
|
||||||
|
base = select(ImageRecord.id).where(
|
||||||
|
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||||
|
)
|
||||||
|
count = session.execute(
|
||||||
|
select(func.count()).select_from(base.subquery())
|
||||||
|
).scalar_one()
|
||||||
|
sample_ids = session.execute(
|
||||||
|
base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP)
|
||||||
|
).scalars().all()
|
||||||
|
return {"count": count, "sample_ids": list(sample_ids)}
|
||||||
|
|
||||||
|
|
||||||
|
def delete_min_dimension_violations(
|
||||||
|
session: Session, *, min_width: int, min_height: int, images_root: Path,
|
||||||
|
) -> int:
|
||||||
|
"""Delete every image_record where width<min_w OR height<min_h.
|
||||||
|
Routes through delete_images so file-unlink + cascading FKs
|
||||||
|
(image_tag / image_provenance / etc.) are handled uniformly."""
|
||||||
|
ids = session.execute(
|
||||||
|
select(ImageRecord.id).where(
|
||||||
|
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
if not ids:
|
||||||
|
return 0
|
||||||
|
result = delete_images(
|
||||||
|
session, image_ids=list(ids), images_root=images_root,
|
||||||
|
)
|
||||||
|
return result["images_deleted"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Audit lifecycle (transparency + single_color async scans).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class AuditAlreadyRunning(Exception):
|
||||||
|
"""Another audit_run is currently in status='running' — wait or
|
||||||
|
cancel it before starting a new one. Surfaces as HTTP 409 in the
|
||||||
|
/api/cleanup/audit POST endpoint."""
|
||||||
|
|
||||||
|
|
||||||
|
class AuditNotReady(Exception):
|
||||||
|
"""apply_audit_run called on an audit whose status is not 'ready'."""
|
||||||
|
|
||||||
|
|
||||||
|
class ConfirmTokenMismatch(Exception):
|
||||||
|
"""Operator-supplied confirm token did not match server-recomputed token."""
|
||||||
|
|
||||||
|
|
||||||
|
_VALID_RULES = ("transparency", "single_color")
|
||||||
|
|
||||||
|
|
||||||
|
def start_audit_run(
|
||||||
|
session: Session, *, rule: str, params: dict[str, Any],
|
||||||
|
) -> int:
|
||||||
|
"""Create a LibraryAuditRun row in status='running' and dispatch the
|
||||||
|
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||||
|
|
||||||
|
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||||
|
has status='running'. Operator must cancel or wait."""
|
||||||
|
if rule not in _VALID_RULES:
|
||||||
|
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||||
|
existing = session.execute(
|
||||||
|
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
raise AuditAlreadyRunning(existing)
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule=rule,
|
||||||
|
params=params,
|
||||||
|
status="running",
|
||||||
|
scanned_count=0,
|
||||||
|
matched_count=0,
|
||||||
|
matched_ids=[],
|
||||||
|
)
|
||||||
|
session.add(audit)
|
||||||
|
session.flush()
|
||||||
|
audit_id = audit.id
|
||||||
|
# Dispatch after flush so audit_id is populated; commit happens in
|
||||||
|
# the API handler so the audit row + dispatch are visible together.
|
||||||
|
from ..tasks.library_audit import scan_library_for_rule
|
||||||
|
scan_library_for_rule.delay(audit_id)
|
||||||
|
return audit_id
|
||||||
|
|
||||||
|
|
||||||
|
def apply_audit_run(
|
||||||
|
session: Session, *, audit_id: int, confirm_token: str, images_root: Path,
|
||||||
|
) -> int:
|
||||||
|
"""Delete all images in audit_run.matched_ids after confirming token.
|
||||||
|
Marks audit status='applied'. Routes through delete_images so files
|
||||||
|
+ cascading FK rows are handled uniformly."""
|
||||||
|
audit = session.execute(
|
||||||
|
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if audit is None:
|
||||||
|
raise ValueError(f"audit_run {audit_id} not found")
|
||||||
|
if audit.status != "ready":
|
||||||
|
raise AuditNotReady(audit.status)
|
||||||
|
# Token format matches modal/DestructiveConfirmModal.vue convention:
|
||||||
|
# ${action}-${kind}-${runId}. The modal hardcodes action ∈ {'restore',
|
||||||
|
# 'delete'}; "apply audit" is semantically a delete of the matched
|
||||||
|
# images, so we use 'delete-audit-<id>' (not 'apply-audit-<id>').
|
||||||
|
expected = f"delete-audit-{audit_id}"
|
||||||
|
if confirm_token != expected:
|
||||||
|
raise ConfirmTokenMismatch(expected)
|
||||||
|
ids = list(audit.matched_ids or [])
|
||||||
|
deleted = 0
|
||||||
|
if ids:
|
||||||
|
result = delete_images(session, image_ids=ids, images_root=images_root)
|
||||||
|
deleted = result["images_deleted"]
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(status="applied", finished_at=datetime.now(UTC))
|
||||||
|
)
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||||
|
"""Flip a running audit_run to 'cancelled'. The scan task checks
|
||||||
|
for status=='cancelled' between batches and exits cleanly."""
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.where(LibraryAuditRun.status == "running")
|
||||||
|
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||||
|
)
|
||||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ..models import Artist, Source
|
from ..models import Artist, Source
|
||||||
@@ -97,20 +98,40 @@ class ExtensionService:
|
|||||||
raise UnknownPlatformError(f"no platform pattern matched {url!r}")
|
raise UnknownPlatformError(f"no platform pattern matched {url!r}")
|
||||||
|
|
||||||
async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]:
|
async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]:
|
||||||
|
"""Race-safe find-or-create on Artist by slug. Mirrors the
|
||||||
|
savepoint + IntegrityError recovery pattern used in
|
||||||
|
Importer._find_or_create_source/post (see
|
||||||
|
reference_scalar_one_or_none_duplicates memory). Without this,
|
||||||
|
two concurrent quick-add-source calls hitting the same artist
|
||||||
|
would both miss the existence check and the second INSERT would
|
||||||
|
500 against uq_artist_slug.
|
||||||
|
"""
|
||||||
slug = slugify(raw_name)
|
slug = slugify(raw_name)
|
||||||
existing = (await self.session.execute(
|
existing = (await self.session.execute(
|
||||||
select(Artist).where(Artist.slug == slug)
|
select(Artist).where(Artist.slug == slug)
|
||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return existing, False
|
return existing, False
|
||||||
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
sp = await self.session.begin_nested()
|
||||||
self.session.add(artist)
|
try:
|
||||||
await self.session.flush()
|
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
||||||
return artist, True
|
self.session.add(artist)
|
||||||
|
await self.session.flush()
|
||||||
|
await sp.commit()
|
||||||
|
return artist, True
|
||||||
|
except IntegrityError:
|
||||||
|
await sp.rollback()
|
||||||
|
recovered = (await self.session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slug)
|
||||||
|
)).scalar_one()
|
||||||
|
return recovered, False
|
||||||
|
|
||||||
async def _find_or_create_source(
|
async def _find_or_create_source(
|
||||||
self, *, artist_id: int, platform: str, url: str,
|
self, *, artist_id: int, platform: str, url: str,
|
||||||
) -> tuple[Source, bool]:
|
) -> tuple[Source, bool]:
|
||||||
|
"""Race-safe — same pattern as _find_or_create_artist above. The
|
||||||
|
uq_source_artist_platform_url constraint catches the duplicate
|
||||||
|
insert; we roll the savepoint back and re-select."""
|
||||||
existing = (await self.session.execute(
|
existing = (await self.session.execute(
|
||||||
select(Source).where(
|
select(Source).where(
|
||||||
Source.artist_id == artist_id,
|
Source.artist_id == artist_id,
|
||||||
@@ -120,8 +141,24 @@ class ExtensionService:
|
|||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return existing, False
|
return existing, False
|
||||||
src = Source(artist_id=artist_id, platform=platform, url=url, enabled=True)
|
sp = await self.session.begin_nested()
|
||||||
self.session.add(src)
|
try:
|
||||||
await self.session.flush()
|
src = Source(
|
||||||
|
artist_id=artist_id, platform=platform,
|
||||||
|
url=url, enabled=True,
|
||||||
|
)
|
||||||
|
self.session.add(src)
|
||||||
|
await self.session.flush()
|
||||||
|
await sp.commit()
|
||||||
|
except IntegrityError:
|
||||||
|
await sp.rollback()
|
||||||
|
recovered = (await self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
)).scalar_one()
|
||||||
|
return recovered, False
|
||||||
await self.session.commit()
|
await self.session.commit()
|
||||||
return src, True
|
return src, True
|
||||||
|
|||||||
@@ -1,26 +1,34 @@
|
|||||||
"""Cursor-paginated gallery queries.
|
"""Cursor-paginated gallery queries.
|
||||||
|
|
||||||
Cursor format: opaque base64-encoded "<iso8601_created_at>:<image_id>".
|
Cursor format: opaque base64-encoded "<iso8601_effective_date>:<image_id>".
|
||||||
Pagination key is (created_at DESC, id DESC) so we don't drift when new
|
|
||||||
imports arrive between page loads. Decoding rejects malformed cursors with
|
Pagination key is (effective_date DESC, id DESC) where effective_date is
|
||||||
a ValueError; the API layer translates that to HTTP 400.
|
COALESCE(post.post_date, image_record.created_at) so the gallery surfaces
|
||||||
|
images by ORIGINAL publish date when known, falling back to FC's scan
|
||||||
|
date. Important for migrated content: ~57k IR images scanned in a single
|
||||||
|
week would otherwise all share the same created_at and pile up in one
|
||||||
|
month bucket. The effective_date spreads them across the years they
|
||||||
|
were originally published.
|
||||||
|
|
||||||
|
Decoding rejects malformed cursors with a ValueError; the API layer
|
||||||
|
translates that to HTTP 400.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import and_, exists, func, or_, select
|
from sqlalchemy import Select, and_, exists, func, or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ..models import Artist, ImageProvenance, ImageRecord, Source, Tag
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
|
|
||||||
CURSOR_SEPARATOR = "|"
|
CURSOR_SEPARATOR = "|"
|
||||||
|
|
||||||
|
|
||||||
def encode_cursor(created_at: datetime, image_id: int) -> str:
|
def encode_cursor(effective_date: datetime, image_id: int) -> str:
|
||||||
raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}"
|
raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}"
|
||||||
return base64.urlsafe_b64encode(raw.encode()).decode()
|
return base64.urlsafe_b64encode(raw.encode()).decode()
|
||||||
|
|
||||||
|
|
||||||
@@ -33,6 +41,26 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
|||||||
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _effective_date_col():
|
||||||
|
"""SQL expression: COALESCE(post.post_date, image_record.created_at).
|
||||||
|
|
||||||
|
Used as the canonical sort/group/filter key across the gallery so
|
||||||
|
images backfilled with primary_post_id (e.g. via tag_apply phase 4)
|
||||||
|
surface at their original publish date, not their FC import date.
|
||||||
|
Images without a Post (or with Post.post_date NULL) fall back to
|
||||||
|
image_record.created_at and still order coherently against
|
||||||
|
post-attached ones.
|
||||||
|
"""
|
||||||
|
return func.coalesce(Post.post_date, ImageRecord.created_at)
|
||||||
|
|
||||||
|
|
||||||
|
def _outer_join_primary_post(stmt: Select) -> Select:
|
||||||
|
"""LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
|
||||||
|
above sees Post.post_date when available. Images without a post
|
||||||
|
survive the join as NULL on the Post side; COALESCE handles it."""
|
||||||
|
return stmt.outerjoin(Post, Post.id == ImageRecord.primary_post_id)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class GalleryImage:
|
class GalleryImage:
|
||||||
id: int
|
id: int
|
||||||
@@ -41,7 +69,9 @@ class GalleryImage:
|
|||||||
mime: str
|
mime: str
|
||||||
width: int | None
|
width: int | None
|
||||||
height: int | None
|
height: int | None
|
||||||
created_at: datetime
|
created_at: datetime # FC's row-insert time
|
||||||
|
effective_date: datetime # COALESCE(post.post_date, created_at)
|
||||||
|
posted_at: datetime | None # post.post_date if known, else None
|
||||||
thumbnail_url: str
|
thumbnail_url: str
|
||||||
artist: dict | None = None
|
artist: dict | None = None
|
||||||
|
|
||||||
@@ -78,7 +108,7 @@ def _require_single_filter(tag_id, post_id, artist_id) -> None:
|
|||||||
def _provenance_clause(post_id, artist_id):
|
def _provenance_clause(post_id, artist_id):
|
||||||
"""Correlated EXISTS clause (NOT a join) so an image with multiple
|
"""Correlated EXISTS clause (NOT a join) so an image with multiple
|
||||||
matching provenance rows is returned exactly once and the
|
matching provenance rows is returned exactly once and the
|
||||||
(created_at DESC, id DESC) cursor ordering is unaffected."""
|
(effective_date DESC, id DESC) cursor ordering is unaffected."""
|
||||||
if post_id is not None:
|
if post_id is not None:
|
||||||
return exists().where(
|
return exists().where(
|
||||||
ImageProvenance.image_record_id == ImageRecord.id,
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
@@ -125,7 +155,9 @@ class GalleryService:
|
|||||||
raise ValueError("limit must be between 1 and 200")
|
raise ValueError("limit must be between 1 and 200")
|
||||||
_require_single_filter(tag_id, post_id, artist_id)
|
_require_single_filter(tag_id, post_id, artist_id)
|
||||||
|
|
||||||
stmt = select(ImageRecord)
|
eff = _effective_date_col()
|
||||||
|
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
||||||
|
stmt = _outer_join_primary_post(stmt)
|
||||||
if tag_id is not None:
|
if tag_id is not None:
|
||||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||||
image_tag.c.tag_id == tag_id
|
image_tag.c.tag_id == tag_id
|
||||||
@@ -138,34 +170,38 @@ class GalleryService:
|
|||||||
cur_ts, cur_id = decode_cursor(cursor)
|
cur_ts, cur_id = decode_cursor(cursor)
|
||||||
stmt = stmt.where(
|
stmt = stmt.where(
|
||||||
or_(
|
or_(
|
||||||
ImageRecord.created_at < cur_ts,
|
eff < cur_ts,
|
||||||
and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id),
|
and_(eff == cur_ts, ImageRecord.id < cur_id),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1)
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1)
|
||||||
rows = (await self.session.execute(stmt)).scalars().all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
|
|
||||||
next_cursor = None
|
next_cursor = None
|
||||||
if len(rows) > limit:
|
if len(rows) > limit:
|
||||||
last = rows[limit - 1]
|
last_record, _last_posted_at, last_eff = rows[limit - 1]
|
||||||
next_cursor = encode_cursor(last.created_at, last.id)
|
next_cursor = encode_cursor(last_eff, last_record.id)
|
||||||
rows = rows[:limit]
|
rows = rows[:limit]
|
||||||
|
|
||||||
artists = await _artists_for(self.session, [r.id for r in rows])
|
artists = await _artists_for(
|
||||||
|
self.session, [r[0].id for r in rows]
|
||||||
|
)
|
||||||
images = [
|
images = [
|
||||||
GalleryImage(
|
GalleryImage(
|
||||||
id=r.id,
|
id=record.id,
|
||||||
path=r.path,
|
path=record.path,
|
||||||
sha256=r.sha256,
|
sha256=record.sha256,
|
||||||
mime=r.mime,
|
mime=record.mime,
|
||||||
width=r.width,
|
width=record.width,
|
||||||
height=r.height,
|
height=record.height,
|
||||||
created_at=r.created_at,
|
created_at=record.created_at,
|
||||||
thumbnail_url=thumbnail_url(r.sha256, r.mime),
|
effective_date=eff_date,
|
||||||
artist=artists.get(r.id),
|
posted_at=posted_at,
|
||||||
|
thumbnail_url=thumbnail_url(record.sha256, record.mime),
|
||||||
|
artist=artists.get(record.id),
|
||||||
)
|
)
|
||||||
for r in rows
|
for record, posted_at, eff_date in rows
|
||||||
]
|
]
|
||||||
return GalleryPage(
|
return GalleryPage(
|
||||||
images=images,
|
images=images,
|
||||||
@@ -179,11 +215,13 @@ class GalleryService:
|
|||||||
post_id: int | None = None,
|
post_id: int | None = None,
|
||||||
artist_id: int | None = None,
|
artist_id: int | None = None,
|
||||||
) -> list[TimelineBucket]:
|
) -> list[TimelineBucket]:
|
||||||
year_col = func.date_part("year", ImageRecord.created_at).label("yr")
|
eff = _effective_date_col()
|
||||||
month_col = func.date_part("month", ImageRecord.created_at).label("mo")
|
year_col = func.date_part("year", eff).label("yr")
|
||||||
|
month_col = func.date_part("month", eff).label("mo")
|
||||||
stmt = select(
|
stmt = select(
|
||||||
year_col, month_col, func.count(ImageRecord.id).label("cnt")
|
year_col, month_col, func.count(ImageRecord.id).label("cnt")
|
||||||
)
|
)
|
||||||
|
stmt = _outer_join_primary_post(stmt)
|
||||||
_require_single_filter(tag_id, post_id, artist_id)
|
_require_single_filter(tag_id, post_id, artist_id)
|
||||||
if tag_id is not None:
|
if tag_id is not None:
|
||||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||||
@@ -201,14 +239,17 @@ class GalleryService:
|
|||||||
post_id: int | None = None, artist_id: int | None = None,
|
post_id: int | None = None, artist_id: int | None = None,
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
"""Returns a cursor that, when passed to scroll(), positions at the
|
"""Returns a cursor that, when passed to scroll(), positions at the
|
||||||
first image of the given year-month. None if the bucket is empty.
|
first image of the given year-month (by effective_date, not
|
||||||
|
created_at). None if the bucket is empty.
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import extract
|
from sqlalchemy import extract
|
||||||
|
|
||||||
stmt = select(ImageRecord).where(
|
eff = _effective_date_col()
|
||||||
extract("year", ImageRecord.created_at) == year,
|
stmt = select(ImageRecord, eff.label("eff")).where(
|
||||||
extract("month", ImageRecord.created_at) == month,
|
extract("year", eff) == year,
|
||||||
|
extract("month", eff) == month,
|
||||||
)
|
)
|
||||||
|
stmt = _outer_join_primary_post(stmt)
|
||||||
_require_single_filter(tag_id, post_id, artist_id)
|
_require_single_filter(tag_id, post_id, artist_id)
|
||||||
if tag_id is not None:
|
if tag_id is not None:
|
||||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||||
@@ -217,13 +258,14 @@ class GalleryService:
|
|||||||
prov = _provenance_clause(post_id, artist_id)
|
prov = _provenance_clause(post_id, artist_id)
|
||||||
if prov is not None:
|
if prov is not None:
|
||||||
stmt = stmt.where(prov)
|
stmt = stmt.where(prov)
|
||||||
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1)
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
||||||
first = (await self.session.execute(stmt)).scalar_one_or_none()
|
first = (await self.session.execute(stmt)).first()
|
||||||
if first is None:
|
if first is None:
|
||||||
return None
|
return None
|
||||||
|
record, eff_date = first
|
||||||
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
|
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
|
||||||
# is the first result in the next scroll().
|
# is the first result in the next scroll().
|
||||||
return encode_cursor(first.created_at, first.id + 1)
|
return encode_cursor(eff_date, record.id + 1)
|
||||||
|
|
||||||
async def get_image_with_tags(self, image_id: int) -> dict | None:
|
async def get_image_with_tags(self, image_id: int) -> dict | None:
|
||||||
record = await self.session.get(ImageRecord, image_id)
|
record = await self.session.get(ImageRecord, image_id)
|
||||||
@@ -236,6 +278,13 @@ class GalleryService:
|
|||||||
.order_by(Tag.kind.asc(), Tag.name.asc())
|
.order_by(Tag.kind.asc(), Tag.name.asc())
|
||||||
)
|
)
|
||||||
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
||||||
|
# Fetch the canonical post.post_date for this image (if any) so
|
||||||
|
# the modal can show "Posted on <date>" alongside import date.
|
||||||
|
posted_at = None
|
||||||
|
if record.primary_post_id is not None:
|
||||||
|
posted_at = (await self.session.execute(
|
||||||
|
select(Post.post_date).where(Post.id == record.primary_post_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
neighbors = await self._neighbors(record)
|
neighbors = await self._neighbors(record)
|
||||||
# Direct artist FK — used by the modal's ProvenancePanel as a
|
# Direct artist FK — used by the modal's ProvenancePanel as a
|
||||||
# fallback when ImageProvenance is empty (i.e., filesystem-
|
# fallback when ImageProvenance is empty (i.e., filesystem-
|
||||||
@@ -256,6 +305,7 @@ class GalleryService:
|
|||||||
"size_bytes": record.size_bytes,
|
"size_bytes": record.size_bytes,
|
||||||
"integrity_status": record.integrity_status,
|
"integrity_status": record.integrity_status,
|
||||||
"created_at": record.created_at.isoformat(),
|
"created_at": record.created_at.isoformat(),
|
||||||
|
"posted_at": posted_at.isoformat() if posted_at else None,
|
||||||
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
|
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
|
||||||
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
||||||
"artist": (
|
"artist": (
|
||||||
@@ -275,34 +325,41 @@ class GalleryService:
|
|||||||
}
|
}
|
||||||
|
|
||||||
async def _neighbors(self, record: ImageRecord) -> dict:
|
async def _neighbors(self, record: ImageRecord) -> dict:
|
||||||
prev_stmt = (
|
# Compute the boundary image's effective_date in Python (one query
|
||||||
select(ImageRecord.id)
|
# below + the SELECT we already have on `record`) and use it for
|
||||||
.where(
|
# the neighbor comparison. Cheaper than re-deriving in SQL via
|
||||||
|
# correlated subquery.
|
||||||
|
boundary_eff = record.created_at
|
||||||
|
if record.primary_post_id is not None:
|
||||||
|
post_date = (await self.session.execute(
|
||||||
|
select(Post.post_date).where(Post.id == record.primary_post_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if post_date is not None:
|
||||||
|
boundary_eff = post_date
|
||||||
|
|
||||||
|
eff = _effective_date_col()
|
||||||
|
prev_stmt = _outer_join_primary_post(
|
||||||
|
select(ImageRecord.id).where(
|
||||||
or_(
|
or_(
|
||||||
ImageRecord.created_at > record.created_at,
|
eff > boundary_eff,
|
||||||
and_(
|
and_(
|
||||||
ImageRecord.created_at == record.created_at,
|
eff == boundary_eff,
|
||||||
ImageRecord.id > record.id,
|
ImageRecord.id > record.id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc())
|
).order_by(eff.asc(), ImageRecord.id.asc()).limit(1)
|
||||||
.limit(1)
|
next_stmt = _outer_join_primary_post(
|
||||||
)
|
select(ImageRecord.id).where(
|
||||||
next_stmt = (
|
|
||||||
select(ImageRecord.id)
|
|
||||||
.where(
|
|
||||||
or_(
|
or_(
|
||||||
ImageRecord.created_at < record.created_at,
|
eff < boundary_eff,
|
||||||
and_(
|
and_(
|
||||||
ImageRecord.created_at == record.created_at,
|
eff == boundary_eff,
|
||||||
ImageRecord.id < record.id,
|
ImageRecord.id < record.id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc())
|
).order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
|
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
|
||||||
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
|
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
|
||||||
return {"prev_id": prev_id, "next_id": next_id}
|
return {"prev_id": prev_id, "next_id": next_id}
|
||||||
@@ -311,9 +368,11 @@ class GalleryService:
|
|||||||
def _group_by_year_month(
|
def _group_by_year_month(
|
||||||
images: list[GalleryImage],
|
images: list[GalleryImage],
|
||||||
) -> list[tuple[int, int, list[int]]]:
|
) -> list[tuple[int, int, list[int]]]:
|
||||||
|
"""Group by effective_date's year/month so migrated content surfaces
|
||||||
|
in the publish-date buckets, not the FC-scan-date bucket."""
|
||||||
groups: list[tuple[int, int, list[int]]] = []
|
groups: list[tuple[int, int, list[int]]] = []
|
||||||
for img in images:
|
for img in images:
|
||||||
y, m = img.created_at.year, img.created_at.month
|
y, m = img.effective_date.year, img.effective_date.month
|
||||||
if groups and groups[-1][0] == y and groups[-1][1] == m:
|
if groups and groups[-1][0] == y and groups[-1][1] == m:
|
||||||
groups[-1][2].append(img.id)
|
groups[-1][2].append(img.id)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import (
|
from ..models import (
|
||||||
@@ -51,7 +52,14 @@ class SkipReason(StrEnum):
|
|||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ImportResult:
|
class ImportResult:
|
||||||
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
# 'imported' — new ImageRecord row created
|
||||||
|
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
|
||||||
|
# 'attached' — non-media saved as PostAttachment
|
||||||
|
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
|
||||||
|
# artist on an already-imported row (no new ImageRecord)
|
||||||
|
# 'skipped' — no work done (true duplicate, too small, etc.)
|
||||||
|
# 'failed' — pipeline error
|
||||||
|
status: str
|
||||||
image_id: int | None = None
|
image_id: int | None = None
|
||||||
skip_reason: SkipReason | None = None
|
skip_reason: SkipReason | None = None
|
||||||
error: str | None = None
|
error: str | None = None
|
||||||
@@ -71,6 +79,31 @@ def is_video(path: Path) -> bool:
|
|||||||
return path.suffix.lower() in VIDEO_EXTS
|
return path.suffix.lower() in VIDEO_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_ext(path: Path) -> str:
|
||||||
|
"""Conservatively extract a file extension for PostAttachment.ext
|
||||||
|
(varchar(32)).
|
||||||
|
|
||||||
|
gallery-dl produces some filenames with URL-encoded query-string
|
||||||
|
artifacts embedded into the basename (e.g.
|
||||||
|
`79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
|
||||||
|
`Path.suffix` finds the LAST dot and returns everything after, which
|
||||||
|
in those cases yields a 50+ char "extension" of mostly base64-ish
|
||||||
|
junk. That blows the column. Operator-flagged 2026-05-25.
|
||||||
|
|
||||||
|
Real extensions are short and alphanumeric. We accept anything ≤ 16
|
||||||
|
chars where every post-dot character is alphanumeric; anything else
|
||||||
|
means the input wasn't a real extension and we return the empty
|
||||||
|
string. ext is nullable-ish (empty string still satisfies NOT NULL)
|
||||||
|
and consumers should treat "" as "no known extension".
|
||||||
|
"""
|
||||||
|
suffix = path.suffix.lower()
|
||||||
|
if not suffix or len(suffix) > 16:
|
||||||
|
return ""
|
||||||
|
if not all(c.isalnum() for c in suffix[1:]):
|
||||||
|
return ""
|
||||||
|
return suffix
|
||||||
|
|
||||||
|
|
||||||
def _mime_for(path: Path) -> str:
|
def _mime_for(path: Path) -> str:
|
||||||
suffix = path.suffix.lower()
|
suffix = path.suffix.lower()
|
||||||
image_mimes = {
|
image_mimes = {
|
||||||
@@ -126,6 +159,179 @@ class Importer:
|
|||||||
self.settings = settings
|
self.settings = settings
|
||||||
self.deep = deep
|
self.deep = deep
|
||||||
self.attachments = AttachmentStore(images_root)
|
self.attachments = AttachmentStore(images_root)
|
||||||
|
# phash near-dup candidate cache. Archive imports call _import_media
|
||||||
|
# per-member; without this cache the per-member SELECT *FROM
|
||||||
|
# image_record WHERE phash IS NOT NULL fetch repeats N times and a
|
||||||
|
# large library × many-member archive blew past soft_time_limit
|
||||||
|
# (300s) — operator-flagged 2026-05-25. Loaded lazily on first
|
||||||
|
# need, appended to on every imported/superseded outcome, never
|
||||||
|
# invalidated mid-Importer (Importer instances are per-task /
|
||||||
|
# per-archive-import so cross-instance staleness is harmless).
|
||||||
|
self._phash_candidates: list[tuple] | None = None
|
||||||
|
|
||||||
|
def _phash_candidates_cache(self) -> list[tuple]:
|
||||||
|
"""Cached `(phash, width, height, id)` rows from image_record.
|
||||||
|
Loaded on first call, appended-to on subsequent imported/
|
||||||
|
superseded outcomes. Soft-timeout pattern: an archive with N
|
||||||
|
members + a library of M existing rows used to do N × M-row
|
||||||
|
fetches (operator-flagged 2026-05-25); now it's exactly one.
|
||||||
|
|
||||||
|
The per-task lifecycle of Importer (instantiated fresh by
|
||||||
|
import_media_file) bounds the cache's staleness window: cross-
|
||||||
|
process changes (other workers importing concurrently) won't
|
||||||
|
be reflected, but that's the same race the un-cached version
|
||||||
|
had — `find_similar` is best-effort anyway."""
|
||||||
|
if self._phash_candidates is None:
|
||||||
|
rows = self.session.execute(
|
||||||
|
select(
|
||||||
|
ImageRecord.phash,
|
||||||
|
ImageRecord.width,
|
||||||
|
ImageRecord.height,
|
||||||
|
ImageRecord.id,
|
||||||
|
).where(ImageRecord.phash.is_not(None))
|
||||||
|
).all()
|
||||||
|
self._phash_candidates = [
|
||||||
|
(r.phash, r.width or 0, r.height or 0, r.id) for r in rows
|
||||||
|
]
|
||||||
|
return self._phash_candidates
|
||||||
|
|
||||||
|
def _phash_cache_append(self, phash, width, height, image_id) -> None:
|
||||||
|
"""Append a freshly-imported row to the cache so subsequent
|
||||||
|
members of the same archive can match against it."""
|
||||||
|
if self._phash_candidates is not None and phash is not None:
|
||||||
|
self._phash_candidates.append(
|
||||||
|
(phash, width or 0, height or 0, image_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _find_or_create_source(
|
||||||
|
self, *, artist_id: int, platform: str, url: str,
|
||||||
|
) -> Source:
|
||||||
|
"""Race-safe find-or-create on `source` keyed by
|
||||||
|
(artist_id, platform, url) — the same key as the
|
||||||
|
`uq_source_artist_platform_url` constraint.
|
||||||
|
|
||||||
|
Two concurrent workers processing different files in the same
|
||||||
|
post can both find no existing Source row then both INSERT,
|
||||||
|
which trips the unique constraint and poisons the session with
|
||||||
|
`psycopg.errors.UniqueViolation`. Operator-flagged 2026-05-26.
|
||||||
|
|
||||||
|
Pattern: select; if absent, open a savepoint and INSERT.
|
||||||
|
On IntegrityError, roll the savepoint back (NOT the outer
|
||||||
|
transaction, which would lose the surrounding scan's progress)
|
||||||
|
and re-select — the concurrent op just created the row we
|
||||||
|
wanted, so the second select will find it.
|
||||||
|
"""
|
||||||
|
existing = self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = Source(artist_id=artist_id, platform=platform, url=url)
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
def _source_for_sidecar(
|
||||||
|
self, *, artist_id: int, platform: str, artist_slug: str,
|
||||||
|
) -> Source:
|
||||||
|
"""Filesystem-import sidecar Source resolver.
|
||||||
|
|
||||||
|
Source represents a subscription feed (one per artist+platform — the
|
||||||
|
gallery-dl URL polled by the FC-3 downloader). The filesystem importer
|
||||||
|
used to call _find_or_create_source(url=sd.post_url), which created
|
||||||
|
one Source row per post URL — 100s of junk Sources per artist, all
|
||||||
|
with enabled=True, polluting the artist detail page and tricking the
|
||||||
|
subscription checker into trying to poll patreon post URLs as feeds.
|
||||||
|
Operator-flagged 2026-05-26.
|
||||||
|
|
||||||
|
New behaviour: if any Source row exists for (artist_id, platform),
|
||||||
|
reuse it regardless of its URL — the artist's real subscription Source
|
||||||
|
(created by the downloader / extension / UI) is the canonical
|
||||||
|
attachment point for filesystem-imported posts. If none exists, create
|
||||||
|
ONE synthetic anchor with url='sidecar:<platform>:<artist_slug>' and
|
||||||
|
enabled=False (so the subscription checker doesn't poll it).
|
||||||
|
"""
|
||||||
|
existing = self.session.execute(
|
||||||
|
select(Source)
|
||||||
|
.where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
)
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
.limit(1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
synthetic_url = f"sidecar:{platform}:{artist_slug}"
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = Source(
|
||||||
|
artist_id=artist_id,
|
||||||
|
platform=platform,
|
||||||
|
url=synthetic_url,
|
||||||
|
enabled=False,
|
||||||
|
)
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(
|
||||||
|
select(Source)
|
||||||
|
.where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
)
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
.limit(1)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
def _find_or_create_post(
|
||||||
|
self, *, source_id: int, external_post_id: str,
|
||||||
|
) -> Post:
|
||||||
|
"""Race-safe find-or-create on `post` keyed by
|
||||||
|
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||||
|
— same savepoint + IntegrityError-recovery pattern."""
|
||||||
|
existing = self.session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = Post(source_id=source_id, external_post_id=external_post_id)
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
def import_one(self, source: Path) -> ImportResult:
|
def import_one(self, source: Path) -> ImportResult:
|
||||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||||
@@ -165,30 +371,13 @@ class Importer:
|
|||||||
return None
|
return None
|
||||||
sd = parse_sidecar(data)
|
sd = parse_sidecar(data)
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
url = sd.post_url or f"sidecar:{platform}"
|
src = self._source_for_sidecar(
|
||||||
src = self.session.execute(
|
artist_id=artist.id, platform=platform, artist_slug=artist.slug,
|
||||||
select(Source).where(
|
)
|
||||||
Source.artist_id == artist.id,
|
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if src is None:
|
|
||||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
||||||
self.session.add(src)
|
|
||||||
self.session.flush()
|
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
post = self.session.execute(
|
return self._find_or_create_post(
|
||||||
select(Post).where(
|
source_id=src.id, external_post_id=epid,
|
||||||
Post.source_id == src.id,
|
)
|
||||||
Post.external_post_id == epid,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if post is None:
|
|
||||||
post = Post(source_id=src.id, external_post_id=epid)
|
|
||||||
self.session.add(post)
|
|
||||||
self.session.flush()
|
|
||||||
return post
|
|
||||||
|
|
||||||
def _capture_attachment(
|
def _capture_attachment(
|
||||||
self, source: Path, *, post: Post | None = None,
|
self, source: Path, *, post: Post | None = None,
|
||||||
@@ -209,7 +398,7 @@ class Importer:
|
|||||||
sha256=sha,
|
sha256=sha,
|
||||||
path=stored,
|
path=stored,
|
||||||
original_filename=source.name,
|
original_filename=source.name,
|
||||||
ext=source.suffix.lower(),
|
ext=_safe_ext(source),
|
||||||
mime=_mime_for(source),
|
mime=_mime_for(source),
|
||||||
size_bytes=source.stat().st_size,
|
size_bytes=source.stat().st_size,
|
||||||
))
|
))
|
||||||
@@ -280,7 +469,18 @@ class Importer:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if self.settings.skip_transparent and has_alpha:
|
if self.settings.skip_transparent and has_alpha:
|
||||||
pct = self._transparency_pct(source)
|
try:
|
||||||
|
pct = self._transparency_pct(source)
|
||||||
|
except OSError as exc:
|
||||||
|
# PIL.verify() at line 263 only validates header structure;
|
||||||
|
# truncated/corrupt pixel data only surfaces when load()
|
||||||
|
# actually decodes (here via getchannel('A')). Convert to
|
||||||
|
# invalid_image skip so the Celery autoretry loop doesn't
|
||||||
|
# bounce the same broken file forever.
|
||||||
|
return ImportResult(
|
||||||
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||||
|
error=f"PIL load failed during transparency check: {exc}",
|
||||||
|
)
|
||||||
if pct >= self.settings.transparency_threshold:
|
if pct >= self.settings.transparency_threshold:
|
||||||
return ImportResult(
|
return ImportResult(
|
||||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||||
@@ -302,21 +502,18 @@ class Importer:
|
|||||||
# Perceptual near-dup (images only; videos keep phash NULL).
|
# Perceptual near-dup (images only; videos keep phash NULL).
|
||||||
phash = None
|
phash = None
|
||||||
if not is_video(source):
|
if not is_video(source):
|
||||||
with Image.open(source) as im:
|
try:
|
||||||
phash = compute_phash(im)
|
with Image.open(source) as im:
|
||||||
|
phash = compute_phash(im)
|
||||||
|
except OSError as exc:
|
||||||
|
# Same rationale as the transparency-check guard above:
|
||||||
|
# broken-pixel-data files pass verify() but blow up here.
|
||||||
|
return ImportResult(
|
||||||
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||||
|
error=f"PIL load failed during phash compute: {exc}",
|
||||||
|
)
|
||||||
if phash is not None:
|
if phash is not None:
|
||||||
cand_rows = self.session.execute(
|
candidates = self._phash_candidates_cache()
|
||||||
select(
|
|
||||||
ImageRecord.phash,
|
|
||||||
ImageRecord.width,
|
|
||||||
ImageRecord.height,
|
|
||||||
ImageRecord.id,
|
|
||||||
).where(ImageRecord.phash.is_not(None))
|
|
||||||
).all()
|
|
||||||
candidates = [
|
|
||||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
|
||||||
for c in cand_rows
|
|
||||||
]
|
|
||||||
rel, match_id = find_similar(
|
rel, match_id = find_similar(
|
||||||
phash, width or 0, height or 0,
|
phash, width or 0, height or 0,
|
||||||
candidates, self.settings.phash_threshold,
|
candidates, self.settings.phash_threshold,
|
||||||
@@ -350,6 +547,7 @@ class Importer:
|
|||||||
)
|
)
|
||||||
self.session.add(record)
|
self.session.add(record)
|
||||||
self.session.flush()
|
self.session.flush()
|
||||||
|
self._phash_cache_append(phash, width, height, record.id)
|
||||||
|
|
||||||
# Folder→artist (anchored to attribution_path).
|
# Folder→artist (anchored to attribution_path).
|
||||||
artist = None
|
artist = None
|
||||||
@@ -373,13 +571,27 @@ class Importer:
|
|||||||
) -> ImportResult:
|
) -> ImportResult:
|
||||||
"""Deep scan: backfill phash/provenance/artist on an
|
"""Deep scan: backfill phash/provenance/artist on an
|
||||||
already-imported record. METADATA ONLY — never re-runs the pHash
|
already-imported record. METADATA ONLY — never re-runs the pHash
|
||||||
near-dup / supersede path. NULL-only, idempotent."""
|
near-dup / supersede path. NULL-only on phash/artist, additive on
|
||||||
|
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
|
||||||
|
Idempotent: a second deep-scan over the same file finds nothing
|
||||||
|
to refresh and is a no-op.
|
||||||
|
|
||||||
|
Returns status="refreshed" so the UI can surface the work done
|
||||||
|
instead of the prior misleading "skipped/duplicate_hash" reading.
|
||||||
|
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
|
||||||
|
as a no-op skip during the original port and the UI showed deep
|
||||||
|
scan as "completed with no changes" even when sidecar metadata
|
||||||
|
actually got re-applied to N existing rows.
|
||||||
|
"""
|
||||||
if existing.phash is None and not is_video(source):
|
if existing.phash is None and not is_video(source):
|
||||||
try:
|
try:
|
||||||
with Image.open(source) as im:
|
with Image.open(source) as im:
|
||||||
ph = compute_phash(im)
|
ph = compute_phash(im)
|
||||||
if ph is not None:
|
if ph is not None:
|
||||||
existing.phash = ph
|
existing.phash = ph
|
||||||
|
# Promoted from NULL to non-NULL → cache is now stale
|
||||||
|
# (this row would newly qualify for the candidates set).
|
||||||
|
self._phash_candidates = None
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("deep rephash failed for %s: %s", source, exc)
|
log.warning("deep rephash failed for %s: %s", source, exc)
|
||||||
|
|
||||||
@@ -392,10 +604,7 @@ class Importer:
|
|||||||
|
|
||||||
self._apply_sidecar(existing, attribution_path, artist)
|
self._apply_sidecar(existing, attribution_path, artist)
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return ImportResult(
|
return ImportResult(status="refreshed", image_id=existing.id)
|
||||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
|
||||||
image_id=existing.id, error="deep: re-derived",
|
|
||||||
)
|
|
||||||
|
|
||||||
def attach_in_place(
|
def attach_in_place(
|
||||||
self,
|
self,
|
||||||
@@ -479,18 +688,7 @@ class Importer:
|
|||||||
except Exception:
|
except Exception:
|
||||||
phash = None
|
phash = None
|
||||||
if phash is not None:
|
if phash is not None:
|
||||||
cand_rows = self.session.execute(
|
candidates = self._phash_candidates_cache()
|
||||||
select(
|
|
||||||
ImageRecord.phash,
|
|
||||||
ImageRecord.width,
|
|
||||||
ImageRecord.height,
|
|
||||||
ImageRecord.id,
|
|
||||||
).where(ImageRecord.phash.is_not(None))
|
|
||||||
).all()
|
|
||||||
candidates = [
|
|
||||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
|
||||||
for c in cand_rows
|
|
||||||
]
|
|
||||||
rel, match_id = find_similar(
|
rel, match_id = find_similar(
|
||||||
phash, width or 0, height or 0,
|
phash, width or 0, height or 0,
|
||||||
candidates, self.settings.phash_threshold,
|
candidates, self.settings.phash_threshold,
|
||||||
@@ -525,6 +723,7 @@ class Importer:
|
|||||||
record.artist_id = artist.id
|
record.artist_id = artist.id
|
||||||
self.session.add(record)
|
self.session.add(record)
|
||||||
self.session.flush()
|
self.session.flush()
|
||||||
|
self._phash_cache_append(phash, width, height, record.id)
|
||||||
|
|
||||||
# Sidecar provenance (best-effort). When `source` is passed, link
|
# Sidecar provenance (best-effort). When `source` is passed, link
|
||||||
# the post to that subscription Source instead of creating a new
|
# the post to that subscription Source instead of creating a new
|
||||||
@@ -619,30 +818,15 @@ class Importer:
|
|||||||
src = explicit_source
|
src = explicit_source
|
||||||
else:
|
else:
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
url = sd.post_url or f"sidecar:{platform}"
|
src = self._source_for_sidecar(
|
||||||
src = self.session.execute(
|
artist_id=artist.id, platform=platform,
|
||||||
select(Source).where(
|
artist_slug=artist.slug,
|
||||||
Source.artist_id == artist.id,
|
)
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if src is None:
|
|
||||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
||||||
self.session.add(src)
|
|
||||||
self.session.flush()
|
|
||||||
|
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
post = self.session.execute(
|
post = self._find_or_create_post(
|
||||||
select(Post).where(
|
source_id=src.id, external_post_id=epid,
|
||||||
Post.source_id == src.id,
|
)
|
||||||
Post.external_post_id == epid,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if post is None:
|
|
||||||
post = Post(source_id=src.id, external_post_id=epid)
|
|
||||||
self.session.add(post)
|
|
||||||
self.session.flush()
|
|
||||||
if sd.post_url is not None:
|
if sd.post_url is not None:
|
||||||
post.post_url = sd.post_url
|
post.post_url = sd.post_url
|
||||||
if sd.post_title is not None:
|
if sd.post_title is not None:
|
||||||
@@ -655,6 +839,15 @@ class Importer:
|
|||||||
post.attachment_count = sd.attachment_count
|
post.attachment_count = sd.attachment_count
|
||||||
post.raw_metadata = sd.raw
|
post.raw_metadata = sd.raw
|
||||||
|
|
||||||
|
# Race-safe (image_record_id, post_id) upsert — mirrors the
|
||||||
|
# _find_or_create_source/post savepoint pattern. The plain
|
||||||
|
# SELECT-then-INSERT pattern lost a race when two workers ran
|
||||||
|
# _apply_sidecar on the same (image, post) pair (e.g. the 5-min
|
||||||
|
# recovery sweep re-enqueued a still-running long import), planting
|
||||||
|
# duplicates that then broke .scalar_one_or_none() on every later
|
||||||
|
# deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
|
||||||
|
# uq_image_provenance_image_post UNIQUE so this savepoint actually
|
||||||
|
# trips on collision.
|
||||||
exists = self.session.execute(
|
exists = self.session.execute(
|
||||||
select(ImageProvenance.id).where(
|
select(ImageProvenance.id).where(
|
||||||
ImageProvenance.image_record_id == record.id,
|
ImageProvenance.image_record_id == record.id,
|
||||||
@@ -662,14 +855,20 @@ class Importer:
|
|||||||
)
|
)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if exists is None:
|
if exists is None:
|
||||||
self.session.add(
|
sp = self.session.begin_nested()
|
||||||
ImageProvenance(
|
try:
|
||||||
image_record_id=record.id,
|
self.session.add(
|
||||||
post_id=post.id,
|
ImageProvenance(
|
||||||
source_id=src.id,
|
image_record_id=record.id,
|
||||||
captured_metadata=sd.raw,
|
post_id=post.id,
|
||||||
|
source_id=src.id,
|
||||||
|
captured_metadata=sd.raw,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
if record.primary_post_id is None:
|
if record.primary_post_id is None:
|
||||||
record.primary_post_id = post.id
|
record.primary_post_id = post.id
|
||||||
self.session.flush()
|
self.session.flush()
|
||||||
@@ -703,6 +902,15 @@ class Importer:
|
|||||||
row id (so tags/series/curation stay attached). ML is cleared so
|
row id (so tags/series/curation stay attached). ML is cleared so
|
||||||
the import task re-derives it on the new pixels.
|
the import task re-derives it on the new pixels.
|
||||||
|
|
||||||
|
After the file swap, the new file's adjacent gallery-dl sidecar
|
||||||
|
(if any) is applied via _apply_sidecar — operator-flagged
|
||||||
|
2026-05-25: scanning a GS download dir with smaller IR-migrated
|
||||||
|
images on the receiving end used to swap files but lose the GS
|
||||||
|
sidecar's post metadata entirely. _apply_sidecar is additive
|
||||||
|
(find-or-create Post / Source / ImageProvenance, NULL-only
|
||||||
|
primary_post_id update) so any pre-existing Post linkage
|
||||||
|
survives untouched.
|
||||||
|
|
||||||
If `new_path` is provided, `source` is assumed to ALREADY be at
|
If `new_path` is provided, `source` is assumed to ALREADY be at
|
||||||
that path (FC-3c attach_in_place case) — skip the copy step.
|
that path (FC-3c attach_in_place case) — skip the copy step.
|
||||||
Otherwise the file is copied via _copy_to_library."""
|
Otherwise the file is copied via _copy_to_library."""
|
||||||
@@ -731,6 +939,26 @@ class Importer:
|
|||||||
# created_at intentionally preserved; updated_at auto-bumps.
|
# created_at intentionally preserved; updated_at auto-bumps.
|
||||||
self.session.flush()
|
self.session.flush()
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
|
# The phash candidate cache (used to avoid N+1 selects during
|
||||||
|
# archive imports) is now stale for `existing.id` — the row's
|
||||||
|
# phash/dimensions changed. Invalidate; the next call re-fetches.
|
||||||
|
self._phash_candidates = None
|
||||||
|
|
||||||
|
# Sidecar enrichment from the new (larger) file's location.
|
||||||
|
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||||
|
# existing row has none, and is internally guarded against
|
||||||
|
# missing-or-malformed sidecars (silent return).
|
||||||
|
try:
|
||||||
|
self._apply_sidecar(existing, source, None)
|
||||||
|
except Exception as exc:
|
||||||
|
# Don't unwind the supersede DB swap if sidecar parsing
|
||||||
|
# blows up unexpectedly — the file replacement is the
|
||||||
|
# critical operation, sidecar is enrichment.
|
||||||
|
log.warning(
|
||||||
|
"sidecar enrichment failed during supersede of "
|
||||||
|
"image_record.id=%s from %s: %s",
|
||||||
|
existing.id, source, exc,
|
||||||
|
)
|
||||||
|
|
||||||
for stale in (old_path, old_thumb):
|
for stale in (old_path, old_thumb):
|
||||||
if not stale or stale == str(dest):
|
if not stale or stale == str(dest):
|
||||||
@@ -746,8 +974,26 @@ class Importer:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def _transparency_pct(self, source: Path) -> float:
|
def _transparency_pct(self, source: Path) -> float:
|
||||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||||
|
|
||||||
|
For animated formats (multi-frame WebP / GIF / APNG), short-circuit
|
||||||
|
to 0.0 instead of decoding every frame. PIL's `getchannel("A")`
|
||||||
|
forces a full decode of all frames in an animated image, which for
|
||||||
|
a large animated WebP takes 5+ minutes and blows past the Celery
|
||||||
|
soft+hard time limits (300s/360s → SIGKILL). Operator-flagged
|
||||||
|
2026-05-26. Transparency analysis on a multi-frame image isn't
|
||||||
|
meaningful for art-curation purposes anyway — different frames
|
||||||
|
have different alpha — so the existing too_transparent skip rule
|
||||||
|
is bypassed entirely for animated content.
|
||||||
|
"""
|
||||||
with Image.open(source) as im:
|
with Image.open(source) as im:
|
||||||
|
if getattr(im, "is_animated", False):
|
||||||
|
log.info(
|
||||||
|
"skipping transparency check for animated image %s "
|
||||||
|
"(n_frames=%d) — avoids multi-frame decode timeout",
|
||||||
|
source, getattr(im, "n_frames", 0),
|
||||||
|
)
|
||||||
|
return 0.0
|
||||||
if im.mode not in ("RGBA", "LA") and not (
|
if im.mode not in ("RGBA", "LA") and not (
|
||||||
im.mode == "P" and "transparency" in im.info
|
im.mode == "P" and "transparency" in im.info
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
"""FC-5 migration tooling.
|
"""FC-5 migration tooling.
|
||||||
|
|
||||||
One module per concern (backup/rollback/gs/ir/overlap/ml_queue/verify).
|
One module per concern (gs/ir/overlap/ml_queue/verify/cleanup).
|
||||||
Each migrator returns a counts dict; the run_migration task wires
|
Each migrator returns a counts dict; the run_migration task wires
|
||||||
that dict into MigrationRun.counts so the UI polling shows progress.
|
that dict into MigrationRun.counts so the UI polling shows progress.
|
||||||
|
|
||||||
|
backup + rollback were retired in FC-3h (2026-05-24); first-class
|
||||||
|
backup lives at backend/app/services/backup_service.py and exposes
|
||||||
|
its own /api/system/backup/* surface.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,146 +0,0 @@
|
|||||||
"""pg_dump + tar.zst-based backup, restorable via pair of subprocess calls.
|
|
||||||
|
|
||||||
Backups live under <images_root>/_backups/. Each backup is two files
|
|
||||||
(SQL + tarball) plus a manifest JSON. Tagged backups (e.g. tag='pre_migration')
|
|
||||||
are how rollback.py finds the most recent restorable snapshot.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
_BACKUPS_DIRNAME = "_backups"
|
|
||||||
|
|
||||||
|
|
||||||
def _libpq_url(sa_url: str) -> str:
|
|
||||||
"""Strip SQLAlchemy driver suffix so pg_dump/psql accept the URL.
|
|
||||||
|
|
||||||
SQLAlchemy uses URLs like `postgresql+psycopg://...` or
|
|
||||||
`postgresql+asyncpg://...`. libpq tools (pg_dump, psql) only know
|
|
||||||
the plain `postgresql://` scheme.
|
|
||||||
"""
|
|
||||||
for driver in ("postgresql+psycopg", "postgresql+asyncpg", "postgresql+psycopg2"):
|
|
||||||
if sa_url.startswith(driver + "://"):
|
|
||||||
return "postgresql://" + sa_url[len(driver) + 3:]
|
|
||||||
return sa_url
|
|
||||||
|
|
||||||
|
|
||||||
def _backups_dir(images_root: Path | None = None) -> Path:
|
|
||||||
# Overridable for tests via monkeypatch.
|
|
||||||
root = images_root if images_root is not None else Path("/images")
|
|
||||||
p = root / _BACKUPS_DIRNAME
|
|
||||||
p.mkdir(parents=True, exist_ok=True)
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
_DEFAULT_SUBPROCESS_TIMEOUT_S = 30 * 60 # 30 minutes
|
|
||||||
|
|
||||||
|
|
||||||
def _run_subprocess(cmd: list[str], **kwargs: Any):
|
|
||||||
# Overridable for tests via monkeypatch. Hard wall-clock timeout
|
|
||||||
# guards against pg_dump / tar / zstd hangs on NFS — without it the
|
|
||||||
# task pretends to be 'running' forever (operator hit this 2026-05-
|
|
||||||
# 23 with two backups stuck in MigrationRun). On timeout
|
|
||||||
# subprocess.run raises TimeoutExpired which the caller surfaces as
|
|
||||||
# a task error.
|
|
||||||
return subprocess.run(
|
|
||||||
cmd,
|
|
||||||
capture_output=True,
|
|
||||||
check=True,
|
|
||||||
timeout=_DEFAULT_SUBPROCESS_TIMEOUT_S,
|
|
||||||
**{k: v for k, v in kwargs.items() if not k.startswith("_")},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def create_backup(
|
|
||||||
*, db_url: str, images_root: Path, tag: str = "manual",
|
|
||||||
) -> dict:
|
|
||||||
"""Create a backup: pg_dump SQL + tar.zst of images.
|
|
||||||
|
|
||||||
Returns a manifest dict. Writes <ts>.sql, <ts>.tar.zst, <ts>.json into
|
|
||||||
<images_root>/_backups/.
|
|
||||||
"""
|
|
||||||
ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
||||||
out_dir = _backups_dir(images_root)
|
|
||||||
sql_path = out_dir / f"fc_{ts}.sql"
|
|
||||||
tar_path = out_dir / f"fc_{ts}.tar.zst"
|
|
||||||
manifest_path = out_dir / f"fc_{ts}.json"
|
|
||||||
|
|
||||||
_run_subprocess(
|
|
||||||
["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), _libpq_url(db_url)],
|
|
||||||
_test_ts=ts,
|
|
||||||
)
|
|
||||||
_run_subprocess(
|
|
||||||
[
|
|
||||||
"tar", "--zstd", "-cf", str(tar_path),
|
|
||||||
"-C", str(images_root.parent), images_root.name,
|
|
||||||
f"--exclude={images_root.name}/_backups",
|
|
||||||
f"--exclude={images_root.name}/_quarantine",
|
|
||||||
],
|
|
||||||
_test_ts=ts,
|
|
||||||
)
|
|
||||||
|
|
||||||
manifest = {
|
|
||||||
"backup_id": ts,
|
|
||||||
"tag": tag,
|
|
||||||
"created_at": datetime.now(UTC).isoformat(),
|
|
||||||
"sql_path": str(sql_path),
|
|
||||||
"tar_path": str(tar_path),
|
|
||||||
}
|
|
||||||
manifest_path.write_text(json.dumps(manifest, indent=2))
|
|
||||||
return manifest
|
|
||||||
|
|
||||||
|
|
||||||
def list_backups(images_root: Path) -> list[dict]:
|
|
||||||
out_dir = _backups_dir(images_root)
|
|
||||||
items = []
|
|
||||||
for mf in sorted(out_dir.glob("fc_*.json"), reverse=True):
|
|
||||||
try:
|
|
||||||
items.append(json.loads(mf.read_text()))
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
def find_latest_backup(images_root: Path, *, tag: str) -> dict | None:
|
|
||||||
for mf in list_backups(images_root):
|
|
||||||
if mf.get("tag") == tag:
|
|
||||||
return mf
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def restore_backup(
|
|
||||||
*, manifest: dict, db_url: str, images_root: Path,
|
|
||||||
) -> dict:
|
|
||||||
"""Restore from a backup manifest.
|
|
||||||
|
|
||||||
1. Replay the .sql via psql.
|
|
||||||
2. Wipe /images/ contents (except _backups/, which holds the file we're using).
|
|
||||||
3. Untar the .tar.zst into /images/.
|
|
||||||
"""
|
|
||||||
sql_path = Path(manifest["sql_path"])
|
|
||||||
tar_path = Path(manifest["tar_path"])
|
|
||||||
|
|
||||||
_run_subprocess(
|
|
||||||
["psql", "-d", _libpq_url(db_url), "-f", str(sql_path)],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup
|
|
||||||
# we're restoring from!).
|
|
||||||
for entry in images_root.iterdir():
|
|
||||||
if entry.name == _BACKUPS_DIRNAME:
|
|
||||||
continue
|
|
||||||
if entry.is_dir():
|
|
||||||
shutil.rmtree(entry)
|
|
||||||
else:
|
|
||||||
entry.unlink()
|
|
||||||
|
|
||||||
_run_subprocess(
|
|
||||||
["tar", "--zstd", "-xf", str(tar_path), "-C", str(images_root.parent)],
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"restored_from": manifest["backup_id"]}
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
"""Restore from the most recent 'pre_migration'-tagged backup."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from . import backup as backup_mod
|
|
||||||
|
|
||||||
|
|
||||||
class NoBackupFoundError(Exception):
|
|
||||||
"""Raised when rollback() is called with no pre_migration backup on disk."""
|
|
||||||
|
|
||||||
|
|
||||||
def rollback_to_pre_migration(*, db_url: str, images_root: Path) -> dict:
|
|
||||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
|
||||||
if manifest is None:
|
|
||||||
raise NoBackupFoundError(
|
|
||||||
"no pre_migration-tagged backup found under <images_root>/_backups/"
|
|
||||||
)
|
|
||||||
return backup_mod.restore_backup(
|
|
||||||
manifest=manifest, db_url=db_url, images_root=images_root,
|
|
||||||
)
|
|
||||||
@@ -37,13 +37,25 @@ from ...utils.slug import slugify
|
|||||||
from .ir_ingest import manifest_path
|
from .ir_ingest import manifest_path
|
||||||
|
|
||||||
# Per-platform artist-profile URL — used as Source.url when restoring
|
# Per-platform artist-profile URL — used as Source.url when restoring
|
||||||
# IR PostMetadata into FC. Keep this table in sync with
|
# IR PostMetadata into FC. Must cover every platform that
|
||||||
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS and
|
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS
|
||||||
# extension/lib/platforms.js.
|
# recognizes; an entry missing here silently drops ALL PostMetadata for
|
||||||
|
# that platform during phase 4 (operator hit this 2026-05-25:
|
||||||
|
# DeviantArt + Pixiv posts in the IR migration produced empty
|
||||||
|
# ImageProvenance because they fell through this table).
|
||||||
|
#
|
||||||
|
# Pixiv caveat: the real profile URL takes a numeric user_id
|
||||||
|
# (https://www.pixiv.net/users/12345), but IR's PostMetadata.artist
|
||||||
|
# stores the display name not the id. We use the slugified name here
|
||||||
|
# so we preserve the artist→post→image linkage; the resulting Source.url
|
||||||
|
# won't resolve in a browser and the operator may want to manually fix
|
||||||
|
# it via Settings → Subscriptions once the migration lands.
|
||||||
_PLATFORM_PROFILE_URL = {
|
_PLATFORM_PROFILE_URL = {
|
||||||
"patreon": "https://www.patreon.com/{slug}",
|
"patreon": "https://www.patreon.com/{slug}",
|
||||||
"subscribestar": "https://www.subscribestar.com/{slug}",
|
"subscribestar": "https://www.subscribestar.com/{slug}",
|
||||||
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
|
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
|
||||||
|
"deviantart": "https://www.deviantart.com/{slug}",
|
||||||
|
"pixiv": "https://www.pixiv.net/users/{slug}",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -110,7 +122,14 @@ async def _ensure_provenance(
|
|||||||
db: AsyncSession, *,
|
db: AsyncSession, *,
|
||||||
image_id: int, post_id: int, source_id: int, dry_run: bool,
|
image_id: int, post_id: int, source_id: int, dry_run: bool,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Returns True if a new ImageProvenance row was inserted."""
|
"""Returns True if a new ImageProvenance row was inserted.
|
||||||
|
|
||||||
|
Also sets ImageRecord.primary_post_id to this post if the image
|
||||||
|
doesn't already have one — preserves any primary_post_id already
|
||||||
|
assigned at download time by the importer (don't clobber). This is
|
||||||
|
the linkage gallery_service.py uses to surface Post.post_date as
|
||||||
|
the image's effective date for sort/group/jump/neighbor nav.
|
||||||
|
"""
|
||||||
existing = (await db.execute(
|
existing = (await db.execute(
|
||||||
select(ImageProvenance.id).where(
|
select(ImageProvenance.id).where(
|
||||||
ImageProvenance.image_record_id == image_id,
|
ImageProvenance.image_record_id == image_id,
|
||||||
@@ -118,6 +137,18 @@ async def _ensure_provenance(
|
|||||||
ImageProvenance.source_id == source_id,
|
ImageProvenance.source_id == source_id,
|
||||||
)
|
)
|
||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
|
|
||||||
|
# Whether-or-not the provenance row already exists, ensure the
|
||||||
|
# image's primary_post_id is set so the gallery date-coalesce works.
|
||||||
|
# Idempotent: only writes when currently NULL.
|
||||||
|
if not dry_run:
|
||||||
|
await db.execute(
|
||||||
|
ImageRecord.__table__.update()
|
||||||
|
.where(ImageRecord.id == image_id)
|
||||||
|
.where(ImageRecord.primary_post_id.is_(None))
|
||||||
|
.values(primary_post_id=post_id)
|
||||||
|
)
|
||||||
|
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return False
|
return False
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
|||||||
@@ -34,10 +34,21 @@ class Embedder:
|
|||||||
if self._model is not None:
|
if self._model is not None:
|
||||||
return
|
return
|
||||||
import torch
|
import torch
|
||||||
from transformers import AutoModel, AutoProcessor
|
from transformers import AutoModel, SiglipImageProcessor
|
||||||
|
|
||||||
self._torch = torch
|
self._torch = torch
|
||||||
self._processor = AutoProcessor.from_pretrained(str(self._model_dir))
|
# FC's embedder only does IMAGE inference — never text. AutoProcessor
|
||||||
|
# loads the full processor including SiglipTokenizer, which requires
|
||||||
|
# the sentencepiece library at import time even if we never call it.
|
||||||
|
# SiglipImageProcessor loads ONLY preprocessor_config.json (image
|
||||||
|
# side) and skips the tokenizer config entirely. Operator hit the
|
||||||
|
# ImportError 2026-05-25 once the ml-worker started actually running
|
||||||
|
# tag_and_embed; switching to the image-only loader avoids the
|
||||||
|
# tokenizer dep without adding ~30 MB of unused C++ build to the
|
||||||
|
# lean ml-worker image.
|
||||||
|
self._processor = SiglipImageProcessor.from_pretrained(
|
||||||
|
str(self._model_dir)
|
||||||
|
)
|
||||||
self._model = AutoModel.from_pretrained(str(self._model_dir))
|
self._model = AutoModel.from_pretrained(str(self._model_dir))
|
||||||
self._model.eval()
|
self._model.eval()
|
||||||
|
|
||||||
|
|||||||
@@ -4,12 +4,14 @@ CPU-only, single-image at a time. Loaded lazily inside the ml-worker
|
|||||||
process; NOT thread-safe — the ml queue worker must run --concurrency=1
|
process; NOT thread-safe — the ml queue worker must run --concurrency=1
|
||||||
(set by the FC-1 entrypoint).
|
(set by the FC-1 entrypoint).
|
||||||
|
|
||||||
Camie's selected_tags.csv columns: tag_id,name,category,count
|
v2 layout reference: HuggingFace Camais03/camie-tagger-v2 root has
|
||||||
where category is a string: general|character|copyright|artist|meta|rating|year
|
camie-tagger-v2.onnx (789 MB) + camie-tagger-v2-metadata.json (7.77 MB)
|
||||||
(unlike WD14's integer Danbooru category ids).
|
+ config.json. Tags ship as nested JSON, not CSV. Preprocessing and
|
||||||
|
output handling follow the published onnx_inference.py reference:
|
||||||
|
ImageNet normalize, NCHW layout, sigmoid on refined logits (output[1]).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import csv
|
import json
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -28,6 +30,8 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
|
|||||||
|
|
||||||
MODEL_NAME = os.environ.get("CAMIE_MODEL_NAME", "camie-tagger-v2")
|
MODEL_NAME = os.environ.get("CAMIE_MODEL_NAME", "camie-tagger-v2")
|
||||||
_MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
|
_MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
|
||||||
|
_MODEL_FILE = f"{MODEL_NAME}.onnx"
|
||||||
|
_METADATA_FILE = f"{MODEL_NAME}-metadata.json"
|
||||||
|
|
||||||
# Below this confidence, predictions aren't stored (keeps the JSON compact).
|
# Below this confidence, predictions aren't stored (keeps the JSON compact).
|
||||||
STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
|
STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
|
||||||
@@ -39,6 +43,12 @@ STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
|
|||||||
# stored at STORE_FLOOR but artist never surfaces.
|
# stored at STORE_FLOOR but artist never surfaces.
|
||||||
SURFACED_CATEGORIES = {"character", "copyright", "general"}
|
SURFACED_CATEGORIES = {"character", "copyright", "general"}
|
||||||
|
|
||||||
|
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
|
||||||
|
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
||||||
|
_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
|
||||||
|
# Square-pad color ≈ ImageNet mean × 255 (matches reference inference).
|
||||||
|
_PAD_COLOR = (124, 116, 104)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class TagPrediction:
|
class TagPrediction:
|
||||||
@@ -51,34 +61,48 @@ class Tagger:
|
|||||||
def __init__(self, model_dir: Path | None = None):
|
def __init__(self, model_dir: Path | None = None):
|
||||||
self._model_dir = model_dir or _MODEL_DIR
|
self._model_dir = model_dir or _MODEL_DIR
|
||||||
self._session = None # onnxruntime.InferenceSession once load()ed
|
self._session = None # onnxruntime.InferenceSession once load()ed
|
||||||
self._tag_meta: list[dict] | None = None
|
self._tag_names: list[str] | None = None
|
||||||
|
self._tag_categories: list[str] | None = None
|
||||||
self._input_name: str | None = None
|
self._input_name: str | None = None
|
||||||
self._output_name: str | None = None
|
self._input_size: int = 512
|
||||||
self._input_size: int = 448
|
|
||||||
|
|
||||||
def load(self) -> None:
|
def load(self) -> None:
|
||||||
if self._session is not None:
|
if self._session is not None:
|
||||||
return
|
return
|
||||||
model_path = self._model_dir / "model.onnx"
|
model_path = self._model_dir / _MODEL_FILE
|
||||||
tags_path = self._model_dir / "selected_tags.csv"
|
meta_path = self._model_dir / _METADATA_FILE
|
||||||
if not model_path.is_file():
|
if not model_path.is_file():
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Camie model.onnx missing at {model_path}. "
|
f"Camie {_MODEL_FILE} missing at {model_path}. "
|
||||||
f"Populate /models via the ml-worker downloader."
|
f"Populate /models via the ml-worker downloader."
|
||||||
)
|
)
|
||||||
if not tags_path.is_file():
|
if not meta_path.is_file():
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Camie selected_tags.csv missing at {tags_path}. "
|
f"Camie {_METADATA_FILE} missing at {meta_path}. "
|
||||||
f"Populate /models via the ml-worker downloader."
|
f"Populate /models via the ml-worker downloader."
|
||||||
)
|
)
|
||||||
|
|
||||||
tag_meta: list[dict] = []
|
with open(meta_path) as f:
|
||||||
with open(tags_path, newline="") as f:
|
metadata = json.load(f)
|
||||||
reader = csv.DictReader(f)
|
|
||||||
for row in reader:
|
# Per Camie v2 onnx_inference.py: idx_to_tag is keyed by str(idx);
|
||||||
tag_meta.append(
|
# tag_to_category maps tag_name -> category. Project to two parallel
|
||||||
{"name": row["name"], "category": row["category"]}
|
# lists indexed by output position for O(1) lookup in the hot path.
|
||||||
)
|
ds = metadata["dataset_info"]
|
||||||
|
idx_to_tag = ds["tag_mapping"]["idx_to_tag"]
|
||||||
|
tag_to_category = ds["tag_mapping"]["tag_to_category"]
|
||||||
|
total = ds["total_tags"]
|
||||||
|
names: list[str] = []
|
||||||
|
cats: list[str] = []
|
||||||
|
for i in range(total):
|
||||||
|
name = idx_to_tag.get(str(i), f"unknown-{i}")
|
||||||
|
names.append(name)
|
||||||
|
cats.append(tag_to_category.get(name, "general"))
|
||||||
|
|
||||||
|
# Input size from metadata; fall back to 512 (the v2 default).
|
||||||
|
self._input_size = int(
|
||||||
|
metadata.get("model_info", {}).get("img_size", 512)
|
||||||
|
)
|
||||||
|
|
||||||
# Lazy import — kept after the file-existence checks so the
|
# Lazy import — kept after the file-existence checks so the
|
||||||
# missing-model RuntimeError still fires first in environments
|
# missing-model RuntimeError still fires first in environments
|
||||||
@@ -89,51 +113,65 @@ class Tagger:
|
|||||||
str(model_path), providers=["CPUExecutionProvider"]
|
str(model_path), providers=["CPUExecutionProvider"]
|
||||||
)
|
)
|
||||||
self._input_name = session.get_inputs()[0].name
|
self._input_name = session.get_inputs()[0].name
|
||||||
self._output_name = session.get_outputs()[0].name
|
|
||||||
input_shape = session.get_inputs()[0].shape
|
|
||||||
for dim in input_shape:
|
|
||||||
if isinstance(dim, int) and dim > 1:
|
|
||||||
self._input_size = dim
|
|
||||||
break
|
|
||||||
# Assign sentinels last so a partial load isn't observable.
|
# Assign sentinels last so a partial load isn't observable.
|
||||||
self._tag_meta = tag_meta
|
self._tag_names = names
|
||||||
|
self._tag_categories = cats
|
||||||
self._session = session
|
self._session = session
|
||||||
|
|
||||||
def _preprocess(self, image_path: Path) -> np.ndarray:
|
def _preprocess(self, image_path: Path) -> np.ndarray:
|
||||||
img = Image.open(image_path)
|
img = Image.open(image_path)
|
||||||
# Camie handles RGBA natively but we still composite onto white so
|
# Composite RGBA onto neutral so transparency doesn't bias the model.
|
||||||
# transparency doesn't bias the model (same as IR's WD14 path).
|
if img.mode == "RGBA":
|
||||||
if img.mode != "RGBA":
|
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
|
||||||
img = img.convert("RGBA")
|
bg.paste(img, mask=img.split()[3])
|
||||||
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
|
img = bg.convert("RGB")
|
||||||
bg.paste(img, mask=img.split()[3])
|
elif img.mode != "RGB":
|
||||||
img = bg.convert("RGB")
|
img = img.convert("RGB")
|
||||||
|
|
||||||
|
# Pad to square with ImageNet-mean color, then bicubic resize.
|
||||||
w, h = img.size
|
w, h = img.size
|
||||||
side = max(w, h)
|
side = max(w, h)
|
||||||
square = Image.new("RGB", (side, side), (255, 255, 255))
|
square = Image.new("RGB", (side, side), _PAD_COLOR)
|
||||||
square.paste(img, ((side - w) // 2, (side - h) // 2))
|
square.paste(img, ((side - w) // 2, (side - h) // 2))
|
||||||
square = square.resize(
|
square = square.resize(
|
||||||
(self._input_size, self._input_size), Image.BICUBIC
|
(self._input_size, self._input_size), Image.BICUBIC
|
||||||
)
|
)
|
||||||
arr = np.array(square, dtype=np.float32)
|
|
||||||
return arr[np.newaxis, :, :, :] # NHWC
|
arr = np.array(square, dtype=np.float32) / 255.0 # HWC, [0,1]
|
||||||
|
arr = (arr - _IMAGENET_MEAN) / _IMAGENET_STD # ImageNet normalize
|
||||||
|
arr = arr.transpose(2, 0, 1) # HWC -> CHW
|
||||||
|
return arr[np.newaxis, :, :, :] # NCHW
|
||||||
|
|
||||||
def infer(self, image_path: Path) -> dict[str, TagPrediction]:
|
def infer(self, image_path: Path) -> dict[str, TagPrediction]:
|
||||||
"""Run Camie on one image. Returns {name: TagPrediction}, only
|
"""Run Camie v2 on one image. Returns {name: TagPrediction} with
|
||||||
entries with confidence >= STORE_FLOOR (across all categories —
|
confidence >= STORE_FLOOR (across all categories — the suggestion
|
||||||
the suggestion service does category filtering later)."""
|
service does category filtering later).
|
||||||
|
|
||||||
|
v2 emits multiple outputs; we use the refined predictions
|
||||||
|
(output[1] per onnx_inference.py). Sigmoid is applied to raw
|
||||||
|
logits to produce [0,1] confidence scores.
|
||||||
|
"""
|
||||||
self.load()
|
self.load()
|
||||||
x = self._preprocess(image_path)
|
x = self._preprocess(image_path)
|
||||||
out = self._session.run([self._output_name], {self._input_name: x})[0][0]
|
outputs = self._session.run(None, {self._input_name: x})
|
||||||
|
# Refined predictions if present (v2 emits initial + refined),
|
||||||
|
# fall back to initial for single-output forks.
|
||||||
|
logits = outputs[1] if len(outputs) > 1 else outputs[0]
|
||||||
|
# Squeeze batch dim, apply sigmoid.
|
||||||
|
probs = 1.0 / (1.0 + np.exp(-logits[0]))
|
||||||
results: dict[str, TagPrediction] = {}
|
results: dict[str, TagPrediction] = {}
|
||||||
for idx, score in enumerate(out):
|
names = self._tag_names
|
||||||
|
cats = self._tag_categories
|
||||||
|
for idx, score in enumerate(probs):
|
||||||
conf = float(score)
|
conf = float(score)
|
||||||
if conf < STORE_FLOOR:
|
if conf < STORE_FLOOR:
|
||||||
continue
|
continue
|
||||||
meta = self._tag_meta[idx]
|
if idx >= len(names):
|
||||||
results[meta["name"]] = TagPrediction(
|
# Output longer than metadata declared — shouldn't happen but
|
||||||
name=meta["name"], category=meta["category"], confidence=conf
|
# don't crash the import pipeline if v2 metadata desynchronizes.
|
||||||
|
continue
|
||||||
|
results[names[idx]] = TagPrediction(
|
||||||
|
name=names[idx], category=cats[idx], confidence=conf
|
||||||
)
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|||||||
@@ -109,7 +109,10 @@ class PostFeedService:
|
|||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
post, artist, source = row
|
post, artist, source = row
|
||||||
thumbs_map = await self._thumbnails_for([post.id])
|
# Detail endpoint returns the FULL image list for PostModal's
|
||||||
|
# masonry grid — feed query still caps at THUMBNAIL_LIMIT via
|
||||||
|
# the default arg.
|
||||||
|
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
||||||
atts_map = await self._attachments_for([post.id])
|
atts_map = await self._attachments_for([post.id])
|
||||||
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
||||||
item["description_full"] = html_to_plain(post.description)
|
item["description_full"] = html_to_plain(post.description)
|
||||||
@@ -117,15 +120,21 @@ class PostFeedService:
|
|||||||
|
|
||||||
# --- composition helpers ---------------------------------------------
|
# --- composition helpers ---------------------------------------------
|
||||||
|
|
||||||
async def _thumbnails_for(self, post_ids: list[int]) -> dict[int, dict]:
|
async def _thumbnails_for(
|
||||||
"""post_id -> {"thumbs": [...up to 6], "more": int}.
|
self, post_ids: list[int], *, limit: int | None = THUMBNAIL_LIMIT,
|
||||||
|
) -> dict[int, dict]:
|
||||||
|
"""post_id -> {"thumbs": [...up to limit], "more": int}.
|
||||||
|
|
||||||
Selects THUMBNAIL_LIMIT+1 images per post via window function so we
|
Selects up to `limit` images per post via window function so we
|
||||||
can detect overflow in a single query.
|
can detect overflow in a single query. Pass `limit=None` to
|
||||||
|
return ALL thumbnails per post (used by `get_post` for PostModal's
|
||||||
|
masonry grid; the feed pass keeps the default cap so payloads
|
||||||
|
stay small).
|
||||||
"""
|
"""
|
||||||
if not post_ids:
|
if not post_ids:
|
||||||
return {}
|
return {}
|
||||||
# Rank images within each post and fetch only the top THUMBNAIL_LIMIT+1.
|
# Rank images within each post; cap at `limit` rows per post when
|
||||||
|
# limit is set, return all when limit is None.
|
||||||
ranked = (
|
ranked = (
|
||||||
select(
|
select(
|
||||||
ImageRecord.id,
|
ImageRecord.id,
|
||||||
@@ -143,12 +152,13 @@ class PostFeedService:
|
|||||||
.where(ImageRecord.primary_post_id.in_(post_ids))
|
.where(ImageRecord.primary_post_id.in_(post_ids))
|
||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
rows = (await self.session.execute(
|
stmt = select(
|
||||||
select(
|
ranked.c.id, ranked.c.primary_post_id,
|
||||||
ranked.c.id, ranked.c.primary_post_id,
|
ranked.c.sha256, ranked.c.mime, ranked.c.total,
|
||||||
ranked.c.sha256, ranked.c.mime, ranked.c.total,
|
)
|
||||||
).where(ranked.c.rn <= THUMBNAIL_LIMIT)
|
if limit is not None:
|
||||||
)).all()
|
stmt = stmt.where(ranked.c.rn <= limit)
|
||||||
|
rows = (await self.session.execute(stmt)).all()
|
||||||
|
|
||||||
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
||||||
for img_id, pid, sha, mime, total in rows:
|
for img_id, pid, sha, mime, total in rows:
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""FC-3k: admin destructive Celery tasks.
|
||||||
|
|
||||||
|
Two long-running ops on the maintenance queue. task_run lifecycle is
|
||||||
|
captured automatically by FC-3i signals — these tasks just return
|
||||||
|
their summary dict so it lands in task_run.metadata (via Celery's
|
||||||
|
result backend) for the dashboard to surface.
|
||||||
|
|
||||||
|
Soft/hard time limits inherit the FC-3i recovery sweep: a runaway
|
||||||
|
task gets killed and flipped to status='timeout' by
|
||||||
|
recover_stalled_task_runs.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..services import cleanup_service
|
||||||
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.delete_artist_cascade_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||||
|
)
|
||||||
|
def delete_artist_cascade_task(self, *, artist_id: int) -> dict:
|
||||||
|
"""Wraps cleanup_service.delete_artist_cascade. Returns the
|
||||||
|
service's summary dict for FC-3i task_run.metadata capture."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
return cleanup_service.delete_artist_cascade(
|
||||||
|
session, artist_id=artist_id, images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.bulk_delete_images_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=900, time_limit=1200, # 15 min / 20 min
|
||||||
|
)
|
||||||
|
def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
|
||||||
|
"""Wraps cleanup_service.delete_images."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
return cleanup_service.delete_images(
|
||||||
|
session, image_ids=image_ids, images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
"""FC-3h: backup/restore Celery tasks.
|
||||||
|
|
||||||
|
All tasks live on the maintenance queue (per celery_app.task_routes).
|
||||||
|
task_run lifecycle tracking is automatic via FC-3i signals — these
|
||||||
|
tasks just record the operator-facing artifact metadata into
|
||||||
|
BackupRun.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..config import get_config
|
||||||
|
from ..models import BackupRun, ImportSettings
|
||||||
|
from ..services import backup_service
|
||||||
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
|
||||||
|
"""Flip a BackupRun row from running/restoring to error with a
|
||||||
|
truncated error message and finished_at. Caller already holds the
|
||||||
|
session open."""
|
||||||
|
row.status = "error"
|
||||||
|
row.error = f"{type(exc).__name__}: {exc}"[:2000]
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.backup_db_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=10, retry_backoff_max=120, max_retries=2,
|
||||||
|
soft_time_limit=600, time_limit=720,
|
||||||
|
)
|
||||||
|
def backup_db_task(self, *, tag: str | None = None,
|
||||||
|
triggered_by: str = "manual") -> dict:
|
||||||
|
"""Create one DB backup. Returns {'backup_run_id': N}."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cfg = get_config()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = BackupRun(
|
||||||
|
kind="db", status="running", tag=tag,
|
||||||
|
triggered_by=triggered_by, started_at=now, manifest={},
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
run_id = row.id
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = backup_service.backup_db(
|
||||||
|
db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
|
||||||
|
tag=tag, triggered_by=triggered_by,
|
||||||
|
)
|
||||||
|
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
if row is not None:
|
||||||
|
_mark_failed(session, row, exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
row.status = "ok"
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
row.sql_path = result["sql_path"]
|
||||||
|
row.size_bytes = result["size_bytes"]
|
||||||
|
row.manifest = {
|
||||||
|
"manifest_path": result["manifest_path"],
|
||||||
|
"ts": result["ts"],
|
||||||
|
}
|
||||||
|
session.commit()
|
||||||
|
return {"backup_run_id": run_id}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.backup_images_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=30, retry_backoff_max=300, max_retries=1,
|
||||||
|
soft_time_limit=21600, time_limit=23400,
|
||||||
|
)
|
||||||
|
def backup_images_task(self, *, tag: str | None = None,
|
||||||
|
triggered_by: str = "manual") -> dict:
|
||||||
|
"""Create one images backup. Same shape as backup_db_task; uses
|
||||||
|
tar_path instead of sql_path."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = BackupRun(
|
||||||
|
kind="images", status="running", tag=tag,
|
||||||
|
triggered_by=triggered_by, started_at=now, manifest={},
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
run_id = row.id
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = backup_service.backup_images(
|
||||||
|
images_root=IMAGES_ROOT,
|
||||||
|
tag=tag, triggered_by=triggered_by,
|
||||||
|
)
|
||||||
|
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
if row is not None:
|
||||||
|
_mark_failed(session, row, exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, run_id)
|
||||||
|
row.status = "ok"
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
row.tar_path = result["tar_path"]
|
||||||
|
row.size_bytes = result["size_bytes"]
|
||||||
|
row.manifest = {
|
||||||
|
"manifest_path": result["manifest_path"],
|
||||||
|
"ts": result["ts"],
|
||||||
|
}
|
||||||
|
session.commit()
|
||||||
|
return {"backup_run_id": run_id}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.restore_db_task",
|
||||||
|
bind=True,
|
||||||
|
max_retries=0, # NEVER auto-retry a half-applied restore.
|
||||||
|
soft_time_limit=1200, time_limit=1800,
|
||||||
|
)
|
||||||
|
def restore_db_task(self, *, source_backup_run_id: int) -> dict:
|
||||||
|
"""Restore from a previous DB backup. Inserts a NEW BackupRun row
|
||||||
|
(kind='db', status='restoring') linked to the source via
|
||||||
|
restored_from_id; flips to 'restored' on success or 'error' on
|
||||||
|
failure. Operator sees the restore as a row in the dashboard."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cfg = get_config()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
src = session.get(BackupRun, source_backup_run_id)
|
||||||
|
if src is None or src.kind != "db" or not src.sql_path:
|
||||||
|
raise ValueError(
|
||||||
|
f"BackupRun id={source_backup_run_id} is not a valid DB backup"
|
||||||
|
)
|
||||||
|
marker = BackupRun(
|
||||||
|
kind="db", status="restoring",
|
||||||
|
triggered_by="restore", started_at=now,
|
||||||
|
restored_from_id=src.id,
|
||||||
|
manifest={"source_sql_path": src.sql_path},
|
||||||
|
)
|
||||||
|
session.add(marker)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(marker)
|
||||||
|
marker_id = marker.id
|
||||||
|
sql_path = src.sql_path
|
||||||
|
|
||||||
|
try:
|
||||||
|
backup_service.restore_db(
|
||||||
|
db_url=cfg.database_url_sync, sql_path=Path(sql_path),
|
||||||
|
)
|
||||||
|
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, marker_id)
|
||||||
|
if row is not None:
|
||||||
|
_mark_failed(session, row, exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, marker_id)
|
||||||
|
row.status = "restored"
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
return {"backup_run_id": marker_id}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.restore_images_task",
|
||||||
|
bind=True, max_retries=0,
|
||||||
|
soft_time_limit=21600, time_limit=23400,
|
||||||
|
)
|
||||||
|
def restore_images_task(self, *, source_backup_run_id: int) -> dict:
|
||||||
|
"""Mirrors restore_db_task; uses backup_service.restore_images."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
src = session.get(BackupRun, source_backup_run_id)
|
||||||
|
if src is None or src.kind != "images" or not src.tar_path:
|
||||||
|
raise ValueError(
|
||||||
|
f"BackupRun id={source_backup_run_id} is not a valid images backup"
|
||||||
|
)
|
||||||
|
marker = BackupRun(
|
||||||
|
kind="images", status="restoring",
|
||||||
|
triggered_by="restore", started_at=now,
|
||||||
|
restored_from_id=src.id,
|
||||||
|
manifest={"source_tar_path": src.tar_path},
|
||||||
|
)
|
||||||
|
session.add(marker)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(marker)
|
||||||
|
marker_id = marker.id
|
||||||
|
tar_path = src.tar_path
|
||||||
|
|
||||||
|
try:
|
||||||
|
backup_service.restore_images(
|
||||||
|
images_root=IMAGES_ROOT, tar_path=Path(tar_path),
|
||||||
|
)
|
||||||
|
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, marker_id)
|
||||||
|
if row is not None:
|
||||||
|
_mark_failed(session, row, exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
row = session.get(BackupRun, marker_id)
|
||||||
|
row.status = "restored"
|
||||||
|
row.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
return {"backup_run_id": marker_id}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.prune_backups",
|
||||||
|
soft_time_limit=300, time_limit=600,
|
||||||
|
)
|
||||||
|
def prune_backups() -> dict:
|
||||||
|
"""Daily Beat. Per-kind retention from ImportSettings.
|
||||||
|
|
||||||
|
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
||||||
|
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||||
|
Status='running' / 'restoring' rows are never pruned (recovery
|
||||||
|
sweep from FC-3i handles those via task_run).
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||||
|
with SessionLocal() as session:
|
||||||
|
s = session.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
for kind, keep in (
|
||||||
|
("db", s.backup_db_keep_last_n),
|
||||||
|
("images", s.backup_images_keep_last_n),
|
||||||
|
):
|
||||||
|
candidates = session.execute(
|
||||||
|
select(BackupRun)
|
||||||
|
.where(BackupRun.kind == kind)
|
||||||
|
.where(BackupRun.tag.is_(None))
|
||||||
|
.where(BackupRun.status.in_(["ok", "error"]))
|
||||||
|
.order_by(BackupRun.started_at.desc())
|
||||||
|
.offset(keep)
|
||||||
|
).scalars().all()
|
||||||
|
for row in candidates:
|
||||||
|
result = backup_service.unlink_artifact_files(
|
||||||
|
sql_path=row.sql_path,
|
||||||
|
tar_path=row.tar_path,
|
||||||
|
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||||
|
)
|
||||||
|
counts["files_unlinked"] += sum(
|
||||||
|
1 for v in result.values() if v
|
||||||
|
)
|
||||||
|
session.delete(row)
|
||||||
|
counts[f"{kind}_deleted"] += 1
|
||||||
|
session.commit()
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.backup.backup_db_nightly",
|
||||||
|
soft_time_limit=60, time_limit=120,
|
||||||
|
)
|
||||||
|
def backup_db_nightly() -> dict:
|
||||||
|
"""Hourly tick. Dispatches a real backup ONLY if the configured
|
||||||
|
UTC hour matches and the nightly setting is enabled. Returns
|
||||||
|
either {'skipped': '<reason>'} or {'dispatched': '<task_id>'}."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
s = session.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
nightly_enabled = s.backup_db_nightly_enabled
|
||||||
|
configured_hour = s.backup_db_nightly_hour_utc
|
||||||
|
if not nightly_enabled:
|
||||||
|
return {"skipped": "nightly disabled"}
|
||||||
|
now_hour = datetime.now(UTC).hour
|
||||||
|
if now_hour != configured_hour:
|
||||||
|
return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
|
||||||
|
res = backup_db_task.delay(triggered_by="nightly")
|
||||||
|
return {"dispatched": res.id}
|
||||||
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
|
|||||||
def _map_result_to_status(result):
|
def _map_result_to_status(result):
|
||||||
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
||||||
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
||||||
'attached' = a non-art file preserved → complete, no ML/thumb."""
|
'attached' = a non-art file preserved → complete, no ML/thumb.
|
||||||
|
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
|
||||||
|
complete, no ML/thumb re-derive (file/pixels unchanged)."""
|
||||||
if result.status in ("imported", "superseded"):
|
if result.status in ("imported", "superseded"):
|
||||||
return ("complete", True)
|
return ("complete", True)
|
||||||
if result.status == "attached":
|
if result.status in ("attached", "refreshed"):
|
||||||
return ("complete", False)
|
return ("complete", False)
|
||||||
if result.status == "skipped":
|
if result.status == "skipped":
|
||||||
return ("skipped", False)
|
return ("skipped", False)
|
||||||
@@ -138,6 +140,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
|||||||
task.result_image_id = result.image_id
|
task.result_image_id = result.image_id
|
||||||
counter_col_name = "imported"
|
counter_col_name = "imported"
|
||||||
counter_col = ImportBatch.imported
|
counter_col = ImportBatch.imported
|
||||||
|
elif result.status == "refreshed":
|
||||||
|
# Deep-scan rederive: existing row got phash/artist/sidecar
|
||||||
|
# refreshed. Task is complete (no further work), but counted in
|
||||||
|
# `refreshed` not `imported` so the UI can surface the actual
|
||||||
|
# work done. operator-flagged 2026-05-25.
|
||||||
|
task.status = "complete"
|
||||||
|
task.result_image_id = result.image_id
|
||||||
|
counter_col_name = "refreshed"
|
||||||
|
counter_col = ImportBatch.refreshed
|
||||||
elif result.status == "attached":
|
elif result.status == "attached":
|
||||||
task.status = "complete"
|
task.status = "complete"
|
||||||
counter_col_name = "attachments"
|
counter_col_name = "attachments"
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
"""scan_library_for_rule Celery task — iterates image_record in keyset-
|
||||||
|
paginated batches, evaluates the audit rule per image, populates
|
||||||
|
LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft
|
||||||
|
time limit (plenty of margin for 100k+ image libraries at ~100ms PIL
|
||||||
|
decode + histogram per image).
|
||||||
|
|
||||||
|
State machine:
|
||||||
|
start: status='running'
|
||||||
|
end success: status='ready'
|
||||||
|
end error: status='error', error=traceback
|
||||||
|
oversize: status='error', error='matched too many images; tighten threshold'
|
||||||
|
external cancel: scan sees status='cancelled' between batches, exits.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import traceback
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..models import ImageRecord, LibraryAuditRun
|
||||||
|
from ..services.audits import single_color, transparency
|
||||||
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BATCH = 500
|
||||||
|
_PROGRESS_TICK = 100
|
||||||
|
_MAX_MATCHED = 50_000
|
||||||
|
|
||||||
|
_RULES = {
|
||||||
|
"transparency": transparency.evaluate,
|
||||||
|
"single_color": single_color.evaluate,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.library_audit.scan_library_for_rule",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
soft_time_limit=7200,
|
||||||
|
time_limit=7500,
|
||||||
|
)
|
||||||
|
def scan_library_for_rule(self, audit_id: int) -> dict:
|
||||||
|
"""See module docstring. Returns a small summary dict for eager-mode
|
||||||
|
test assertions (real workers ignore the return value)."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
try:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
audit = session.get(LibraryAuditRun, audit_id)
|
||||||
|
if audit is None:
|
||||||
|
return {"audit_id": audit_id, "status": "missing"}
|
||||||
|
evaluate = _RULES.get(audit.rule)
|
||||||
|
if evaluate is None:
|
||||||
|
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
||||||
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
|
params = dict(audit.params or {})
|
||||||
|
matched: list[int] = []
|
||||||
|
scanned = 0
|
||||||
|
last_id = 0
|
||||||
|
while True:
|
||||||
|
# Cancellation check between batches.
|
||||||
|
current_status = session.execute(
|
||||||
|
select(LibraryAuditRun.status)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
if current_status == "cancelled":
|
||||||
|
return {"audit_id": audit_id, "status": "cancelled"}
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.path)
|
||||||
|
.where(ImageRecord.id > last_id)
|
||||||
|
.where(ImageRecord.mime.like("image/%"))
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(_BATCH)
|
||||||
|
).all()
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for image_id, image_path in rows:
|
||||||
|
last_id = image_id
|
||||||
|
scanned += 1
|
||||||
|
try:
|
||||||
|
with Image.open(image_path) as im:
|
||||||
|
try:
|
||||||
|
if evaluate(im, **params):
|
||||||
|
matched.append(image_id)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(
|
||||||
|
"audit %s: rule evaluate failed on %s: %s",
|
||||||
|
audit_id, image_path, exc,
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
log.warning(
|
||||||
|
"audit %s: image_record %s file missing at %s; skipping",
|
||||||
|
audit_id, image_id, image_path,
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning(
|
||||||
|
"audit %s: PIL load failed for %s: %s",
|
||||||
|
audit_id, image_path, exc,
|
||||||
|
)
|
||||||
|
if len(matched) > _MAX_MATCHED:
|
||||||
|
_mark_error(
|
||||||
|
session, audit_id,
|
||||||
|
f"matched > {_MAX_MATCHED} images; "
|
||||||
|
"tighten threshold and re-run",
|
||||||
|
)
|
||||||
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
|
if scanned % _PROGRESS_TICK == 0:
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(scanned_count=scanned)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
# Final state.
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(
|
||||||
|
scanned_count=scanned,
|
||||||
|
matched_count=len(matched),
|
||||||
|
matched_ids=matched,
|
||||||
|
status="ready",
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {
|
||||||
|
"audit_id": audit_id,
|
||||||
|
"status": "ready",
|
||||||
|
"scanned": scanned,
|
||||||
|
"matched": len(matched),
|
||||||
|
}
|
||||||
|
except SoftTimeLimitExceeded:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
_mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)")
|
||||||
|
raise
|
||||||
|
except (OperationalError, DBAPIError):
|
||||||
|
# Retryable per the decorator; leave row in 'running' and let
|
||||||
|
# autoretry try again. Recovery sweep catches if all retries fail.
|
||||||
|
raise
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
_mark_error(session, audit_id, tb)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(
|
||||||
|
status="error",
|
||||||
|
error=error_msg,
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
@@ -9,55 +9,94 @@ from PIL import Image
|
|||||||
from sqlalchemy import delete, select, update
|
from sqlalchemy import delete, select, update
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask
|
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
||||||
from ..utils.phash import compute_phash
|
from ..utils.phash import compute_phash
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
STUCK_THRESHOLD_MINUTES = 5
|
STUCK_THRESHOLD_MINUTES = 5
|
||||||
|
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
||||||
OLD_TASK_DAYS = 7
|
OLD_TASK_DAYS = 7
|
||||||
PHASH_PAGE = 500
|
PHASH_PAGE = 500
|
||||||
VERIFY_PAGE = 200
|
VERIFY_PAGE = 200
|
||||||
FFPROBE_TIMEOUT_SECONDS = 10
|
FFPROBE_TIMEOUT_SECONDS = 10
|
||||||
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||||
def recover_interrupted_tasks() -> int:
|
def recover_interrupted_tasks() -> int:
|
||||||
"""Find ImportTask rows stuck in 'processing' for >5 min and re-queue them.
|
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||||
|
|
||||||
Why 5 min: import_media_file is sub-second for the vast majority of
|
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
|
||||||
files; even a large-video transcode caps at the per-task soft_time_limit
|
.delay() and let the import retry. Was 30 min historically;
|
||||||
(5 min) defined on the task itself. Anything still 'processing' after
|
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
|
||||||
that window is a confirmed crash (worker died, DB disconnect mid-flush,
|
import_media_file is sub-second for the vast majority of files and
|
||||||
OOM) and must be recycled. Was 30 min historically; tightened
|
capped at the per-task soft_time_limit (5 min), so anything still
|
||||||
2026-05-24 after operator hit a 2224-row zombie pile during the IR
|
'processing' after that window is a confirmed crash.
|
||||||
migration scan.
|
|
||||||
|
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||||
|
creates rows with status='pending' (commit), then in a second pass
|
||||||
|
transitions to 'queued' and calls .delay() (commit). If the scanner
|
||||||
|
crashes between those two commits, rows are orphaned in 'pending'
|
||||||
|
(never enqueued) with no recovery path — invisible to the
|
||||||
|
'processing' sweep above. Flagged 2026-05-25 by operator hitting a
|
||||||
|
5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
|
||||||
|
the operator drains them via /api/import/retry-failed at their own
|
||||||
|
pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
|
||||||
|
import worker.
|
||||||
|
|
||||||
|
Returns total rows touched (recovered + marked failed).
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
now = datetime.now(UTC)
|
||||||
|
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
stuck_ids = session.execute(
|
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
||||||
select(ImportTask.id)
|
# blew past psycopg's 65535-parameter ceiling once a sweep covered
|
||||||
.where(ImportTask.status == "processing")
|
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
||||||
.where(ImportTask.started_at < cutoff)
|
# /import deep scan piled up orphans). Folding the SELECT into the
|
||||||
).scalars().all()
|
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
||||||
|
# exactly the ids that flipped so the stuck sweep can still
|
||||||
if not stuck_ids:
|
# .delay() each one.
|
||||||
return 0
|
stuck_result = session.execute(
|
||||||
|
|
||||||
session.execute(
|
|
||||||
update(ImportTask)
|
update(ImportTask)
|
||||||
.where(ImportTask.id.in_(stuck_ids))
|
.where(ImportTask.status == "processing")
|
||||||
.values(status="queued", started_at=None, error="recovered from stuck state")
|
.where(ImportTask.started_at < processing_cutoff)
|
||||||
|
.values(
|
||||||
|
status="queued",
|
||||||
|
started_at=None,
|
||||||
|
error="recovered from stuck state",
|
||||||
|
)
|
||||||
|
.returning(ImportTask.id)
|
||||||
)
|
)
|
||||||
|
stuck_ids = [row[0] for row in stuck_result.all()]
|
||||||
|
|
||||||
|
orphan_result = session.execute(
|
||||||
|
update(ImportTask)
|
||||||
|
.where(ImportTask.status.in_(["pending", "queued"]))
|
||||||
|
.where(ImportTask.created_at < orphan_cutoff)
|
||||||
|
.values(
|
||||||
|
status="failed",
|
||||||
|
error=(
|
||||||
|
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||||
|
"(scanner likely crashed mid-enqueue); retry via "
|
||||||
|
"/api/import/retry-failed"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
orphan_count = orphan_result.rowcount or 0
|
||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
from .import_file import import_media_file
|
if stuck_ids:
|
||||||
for tid in stuck_ids:
|
from .import_file import import_media_file
|
||||||
import_media_file.delay(tid)
|
for tid in stuck_ids:
|
||||||
|
import_media_file.delay(tid)
|
||||||
|
|
||||||
return len(stuck_ids)
|
return len(stuck_ids) + orphan_count
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||||
@@ -80,6 +119,71 @@ def cleanup_old_tasks() -> int:
|
|||||||
return result.rowcount or 0
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
|
def recover_stalled_task_runs() -> int:
|
||||||
|
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
||||||
|
to 'error'. FC-3i.
|
||||||
|
|
||||||
|
A row gets stuck when the worker dies without emitting
|
||||||
|
task_postrun / task_failure (e.g. OOM, container restart between
|
||||||
|
signals, signal handler raised+logged). Shares the 5-min threshold
|
||||||
|
with recover_interrupted_tasks for consistency.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
update(TaskRun)
|
||||||
|
.where(TaskRun.status == "running")
|
||||||
|
.where(TaskRun.started_at < cutoff)
|
||||||
|
.values(
|
||||||
|
status="error",
|
||||||
|
error_type="RecoverySweep",
|
||||||
|
error_message=(
|
||||||
|
f"no completion signal received within "
|
||||||
|
f"{STUCK_THRESHOLD_MINUTES} min"
|
||||||
|
),
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||||
|
def prune_task_runs() -> dict:
|
||||||
|
"""Daily retention for task_run rows. FC-3i.
|
||||||
|
|
||||||
|
- 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
|
||||||
|
Success is high-volume, not interesting after a day.
|
||||||
|
- 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
|
||||||
|
(7 days default). Failures are operationally interesting longer.
|
||||||
|
- 'running' rows: NEVER deleted by this task. The recovery sweep
|
||||||
|
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||||
|
terminal state; prune doesn't touch in-flight state.
|
||||||
|
- 'retry' rows: treated as failures (>7d).
|
||||||
|
|
||||||
|
Returns dict of how many rows were deleted in each bucket.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||||
|
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
ok_deleted = session.execute(
|
||||||
|
delete(TaskRun)
|
||||||
|
.where(TaskRun.status == "ok")
|
||||||
|
.where(TaskRun.finished_at < ok_cutoff)
|
||||||
|
).rowcount or 0
|
||||||
|
fail_deleted = session.execute(
|
||||||
|
delete(TaskRun)
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||||
|
.where(TaskRun.finished_at < fail_cutoff)
|
||||||
|
).rowcount or 0
|
||||||
|
session.commit()
|
||||||
|
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
||||||
def backfill_phash() -> int:
|
def backfill_phash() -> int:
|
||||||
"""Recompute phash for stored images that have none (imported before
|
"""Recompute phash for stored images that have none (imported before
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ Dispatches to the right migrator based on `kind`. Updates MigrationRun
|
|||||||
row's status/counts/finished_at as it runs. Failures set status='error'
|
row's status/counts/finished_at as it runs. Failures set status='error'
|
||||||
with the error message preserved.
|
with the error message preserved.
|
||||||
|
|
||||||
kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback, cleanup
|
kinds: gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup
|
||||||
|
(backup + rollback retired 2026-05-24 → see /api/system/backup/*)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -20,10 +21,8 @@ from ..celery_app import celery
|
|||||||
from ..config import get_config
|
from ..config import get_config
|
||||||
from ..models import MigrationRun
|
from ..models import MigrationRun
|
||||||
from ..services.credential_crypto import CredentialCrypto
|
from ..services.credential_crypto import CredentialCrypto
|
||||||
from ..services.migrators import backup as backup_mod
|
|
||||||
from ..services.migrators import cleanup as cleanup_mod
|
from ..services.migrators import cleanup as cleanup_mod
|
||||||
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
||||||
from ..services.migrators import rollback as rollback_mod
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -65,21 +64,11 @@ async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
|||||||
async with factory() as db:
|
async with factory() as db:
|
||||||
await _update_run(db, run_id, status="running")
|
await _update_run(db, run_id, status="running")
|
||||||
try:
|
try:
|
||||||
if kind == "backup":
|
if kind in ("backup", "rollback"):
|
||||||
manifest = backup_mod.create_backup(
|
raise ValueError(
|
||||||
db_url=get_config().database_url_sync,
|
f"kind {kind!r} retired in FC-3h; "
|
||||||
images_root=IMAGES_ROOT,
|
"use /api/system/backup/* instead"
|
||||||
tag=params.get("tag", "manual"),
|
|
||||||
)
|
)
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": 0, "rows_inserted": 0,
|
|
||||||
"rows_skipped": 0, "files_copied": 0,
|
|
||||||
"bytes_copied": 0, "conflicts": 0},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"manifest": manifest},
|
|
||||||
)
|
|
||||||
return manifest
|
|
||||||
|
|
||||||
elif kind == "gs_ingest":
|
elif kind == "gs_ingest":
|
||||||
fc_crypto = CredentialCrypto(_KEY_PATH)
|
fc_crypto = CredentialCrypto(_KEY_PATH)
|
||||||
@@ -166,21 +155,6 @@ async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
|||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
elif kind == "rollback":
|
|
||||||
result = rollback_mod.rollback_to_pre_migration(
|
|
||||||
db_url=get_config().database_url_sync,
|
|
||||||
images_root=IMAGES_ROOT,
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": 0, "rows_inserted": 0,
|
|
||||||
"rows_skipped": 0, "files_copied": 0,
|
|
||||||
"bytes_copied": 0, "conflicts": 0},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"rollback_result": result},
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"unknown kind: {kind}")
|
raise ValueError(f"unknown kind: {kind}")
|
||||||
|
|
||||||
|
|||||||
@@ -59,16 +59,25 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
session.flush()
|
session.flush()
|
||||||
batch_id = batch.id
|
batch_id = batch.id
|
||||||
|
|
||||||
# Skip-set: any source_path that already has a non-failed ImportTask
|
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
|
||||||
# row. Re-running scan_directory must not re-enqueue files the
|
#
|
||||||
# importer has already handled (or is currently handling); doing so
|
# quick: any non-failed prior ImportTask (active OR finished) is
|
||||||
# creates duplicate work and inflates the queue. Failed prior tasks
|
# skipped — quick scan only does new-file enqueue, so re-touching
|
||||||
# are eligible for retry.
|
# already-imported files is wasted work.
|
||||||
|
#
|
||||||
|
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
|
||||||
|
# are skipped. Completed and skipped tasks ARE re-queued because
|
||||||
|
# deep scan exists precisely to re-touch already-imported files
|
||||||
|
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
|
||||||
|
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
|
||||||
|
active_statuses = ["pending", "queued", "processing"]
|
||||||
|
if mode == "deep":
|
||||||
|
skip_statuses = active_statuses
|
||||||
|
else:
|
||||||
|
skip_statuses = active_statuses + ["complete", "skipped"]
|
||||||
non_failed_existing = set(session.execute(
|
non_failed_existing = set(session.execute(
|
||||||
select(ImportTask.source_path).where(
|
select(ImportTask.source_path).where(
|
||||||
ImportTask.status.in_(
|
ImportTask.status.in_(skip_statuses),
|
||||||
["pending", "queued", "processing", "complete", "skipped"]
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
).scalars().all())
|
).scalars().all())
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,30 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
|||||||
|
|
||||||
IMAGES_ROOT = Path("/images")
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
||||||
|
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _thumb_is_valid(path: Path) -> bool:
|
||||||
|
"""Return True iff `path` exists and starts with a JPEG or PNG magic header.
|
||||||
|
|
||||||
|
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for
|
||||||
|
opaque sources, PNG for alpha sources. Anything else (missing file, OSError,
|
||||||
|
truncated, wrong magic) is invalid.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with path.open("rb") as f:
|
||||||
|
head = f.read(12)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if len(head) < 8:
|
||||||
|
return False
|
||||||
|
if head[:3] == THUMB_MAGIC_JPEG:
|
||||||
|
return True
|
||||||
|
if head[:8] == THUMB_MAGIC_PNG:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
||||||
@@ -50,3 +74,58 @@ def generate_thumbnail(self, image_id: int) -> dict:
|
|||||||
session.add(record)
|
session.add(record)
|
||||||
session.commit()
|
session.commit()
|
||||||
return {"status": "ok", "image_id": image_id, "path": str(result.path)}
|
return {"status": "ok", "image_id": image_id, "path": str(result.path)}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.thumbnail.backfill_thumbnails",
|
||||||
|
bind=True,
|
||||||
|
)
|
||||||
|
def backfill_thumbnails(self) -> dict:
|
||||||
|
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
||||||
|
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
||||||
|
|
||||||
|
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for
|
||||||
|
rows that point at a missing or corrupt file before enqueueing — keeps
|
||||||
|
the DB self-consistent on partial runs and makes re-runs safe.
|
||||||
|
|
||||||
|
Returns {"enqueued": N, "ok": M, "regenerated": K} where:
|
||||||
|
- enqueued = total generate_thumbnail.delay() calls
|
||||||
|
- ok = rows whose existing thumbnail file is valid (skipped)
|
||||||
|
- regenerated = subset of enqueued that had a non-NULL thumbnail_path
|
||||||
|
cleared (i.e. missing + corrupt)
|
||||||
|
"""
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
enqueued = 0
|
||||||
|
ok = 0
|
||||||
|
regenerated = 0
|
||||||
|
last_id = 0
|
||||||
|
with SessionLocal() as session:
|
||||||
|
while True:
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
||||||
|
.where(ImageRecord.id > last_id)
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(500)
|
||||||
|
).all()
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for image_id, thumb_path in rows:
|
||||||
|
if thumb_path is None:
|
||||||
|
generate_thumbnail.delay(image_id)
|
||||||
|
enqueued += 1
|
||||||
|
elif _thumb_is_valid(Path(thumb_path)):
|
||||||
|
ok += 1
|
||||||
|
else:
|
||||||
|
session.execute(
|
||||||
|
update(ImageRecord)
|
||||||
|
.where(ImageRecord.id == image_id)
|
||||||
|
.values(thumbnail_path=None)
|
||||||
|
)
|
||||||
|
generate_thumbnail.delay(image_id)
|
||||||
|
enqueued += 1
|
||||||
|
regenerated += 1
|
||||||
|
session.commit()
|
||||||
|
last_id = rows[-1][0]
|
||||||
|
return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated}
|
||||||
|
|||||||
@@ -13,8 +13,21 @@ HASH_SIZE = 8
|
|||||||
|
|
||||||
def compute_phash(pil_image) -> str | None:
|
def compute_phash(pil_image) -> str | None:
|
||||||
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
||||||
failure (videos/unreadable/non-image)."""
|
failure (videos/unreadable/non-image).
|
||||||
|
|
||||||
|
For animated images (multi-frame WebP/GIF/APNG), explicitly seek to
|
||||||
|
frame 0 first. Without this, some PIL operations downstream of
|
||||||
|
imagehash.phash (convert("L"), resize) can iterate all frames and
|
||||||
|
blow past Celery's hard time limit on large animations
|
||||||
|
(operator-flagged 2026-05-26 against animated WebPs). The pHash of
|
||||||
|
frame 0 is the conventional choice for animated content.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
|
if getattr(pil_image, "is_animated", False):
|
||||||
|
try:
|
||||||
|
pil_image.seek(0)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ No per-platform branching: a small common key set with fallbacks; the
|
|||||||
full JSON is kept in raw so anything unmapped is recoverable later.
|
full JSON is kept in raw so anything unmapped is recoverable later.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -21,10 +22,28 @@ class SidecarData:
|
|||||||
raw: dict
|
raw: dict
|
||||||
|
|
||||||
|
|
||||||
|
# gallery-dl prefixes media filenames with `NN_` for in-post ordering
|
||||||
|
# (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar
|
||||||
|
# under the attachment's stem WITHOUT that ordering prefix
|
||||||
|
# (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars.
|
||||||
|
# Confirmed against real Patreon downloads 2026-05-26 — without this
|
||||||
|
# strip, every gallery-dl post-level sidecar was invisible to FC since
|
||||||
|
# FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB).
|
||||||
|
_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
|
||||||
|
|
||||||
|
|
||||||
def find_sidecar(media: Path) -> Path | None:
|
def find_sidecar(media: Path) -> Path | None:
|
||||||
|
# Attachment-level sidecars (image.jpg.json, image.json).
|
||||||
for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
|
for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
|
||||||
if cand.is_file():
|
if cand.is_file():
|
||||||
return cand
|
return cand
|
||||||
|
# gallery-dl post-numbered convention: strip the `NN_` prefix from
|
||||||
|
# the stem and look for that.json in the same directory.
|
||||||
|
m = _NUMBERING_PREFIX.match(media.stem)
|
||||||
|
if m:
|
||||||
|
cand = media.parent / f"{m.group(1)}.json"
|
||||||
|
if cand.is_file():
|
||||||
|
return cand
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Parse the user-facing `kind:name` shortcut used by the add-tag input.
|
||||||
|
|
||||||
|
Mirrors IR's app/utils/tag_prefix.py. Tag.name in FC is stored bare;
|
||||||
|
the `kind:` prefix only exists as an input convention at user-facing
|
||||||
|
places (image-modal add-tag input, future bulk-add forms). The parser
|
||||||
|
is the single owner of the kind-string list — anything not in
|
||||||
|
KNOWN_KINDS keeps its colon as literal text.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
# Kinds the user can type as a prefix at the input boundary.
|
||||||
|
# Exclusions:
|
||||||
|
# - `general` is the default for un-prefixed input (never typed as prefix)
|
||||||
|
# - `archive`, `post` are system-managed
|
||||||
|
# - `artist` was retired in FC-2d-vii-c — artists are first-class
|
||||||
|
# entities (Artist row + ImageRecord.artist_id), browsed via the
|
||||||
|
# provenance axis rather than as tags. See project_provenance_separation.
|
||||||
|
# - `meta`, `rating` retired as user-typeable per operator 2026-05-26 —
|
||||||
|
# content classification only needs character/fandom/series.
|
||||||
|
KNOWN_KINDS: frozenset[str] = frozenset({
|
||||||
|
"character",
|
||||||
|
"fandom",
|
||||||
|
"series",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def parse_kind_prefix(raw: str) -> tuple[str | None, str]:
|
||||||
|
"""Split a raw user-typed tag string into (kind, name).
|
||||||
|
|
||||||
|
Returns (kind, name) where kind is lowercase canonical and in
|
||||||
|
KNOWN_KINDS, or (None, raw.strip()) if no recognized prefix is
|
||||||
|
present. `name` is always whitespace-stripped.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
parse_kind_prefix("character:Saber") -> ("character", "Saber")
|
||||||
|
parse_kind_prefix("Character:Saber") -> ("character", "Saber")
|
||||||
|
parse_kind_prefix("sunset") -> (None, "sunset")
|
||||||
|
parse_kind_prefix("http://example") -> (None, "http://example")
|
||||||
|
parse_kind_prefix("fandom: FSN ") -> ("fandom", "FSN")
|
||||||
|
"""
|
||||||
|
if ":" in raw:
|
||||||
|
prefix, rest = raw.split(":", 1)
|
||||||
|
if prefix.lower() in KNOWN_KINDS:
|
||||||
|
return prefix.lower(), rest.strip()
|
||||||
|
return None, raw.strip()
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "FabledCurator",
|
"name": "FabledCurator",
|
||||||
"version": "1.0.0",
|
"version": "1.0.4",
|
||||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||||
|
|
||||||
"browser_specific_settings": {
|
"browser_specific_settings": {
|
||||||
@@ -11,6 +11,11 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
"content_security_policy": {
|
||||||
|
"_comment": "Override the MV3 default CSP to OMIT upgrade-insecure-requests. FC runs over plain HTTP per the homelab posture (feedback_homelab_http), and the default MV3 CSP would silently upgrade every fetch(http://curator.../...) to https:// and fail with NS_ERROR_GENERATE_FAILURE. Operator-flagged 2026-05-26 after the 'Test connection' button errored despite a working CORS preflight on the backend.",
|
||||||
|
"extension_pages": "script-src 'self'; object-src 'self';"
|
||||||
|
},
|
||||||
|
|
||||||
"permissions": [
|
"permissions": [
|
||||||
"cookies",
|
"cookies",
|
||||||
"storage",
|
"storage",
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"name": "fabledcurator-extension",
|
"name": "fabledcurator-extension",
|
||||||
"version": "1.0.0",
|
"version": "1.0.4",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Firefox extension for FabledCurator",
|
"description": "Firefox extension for FabledCurator",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"lint": "web-ext lint --source-dir=.",
|
"lint": "web-ext lint --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore",
|
||||||
"start": "web-ext run --source-dir=. --firefox=firefox",
|
"start": "web-ext run --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --firefox=firefox",
|
||||||
"build": "web-ext build --source-dir=. --overwrite-dest",
|
"build": "web-ext build --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --overwrite-dest",
|
||||||
"sign": "web-ext sign --source-dir=. --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"web-ext": "^8.0.0"
|
"web-ext": "^8.0.0"
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
sourceDir: '.',
|
|
||||||
artifactsDir: './web-ext-artifacts',
|
|
||||||
ignoreFiles: [
|
|
||||||
'package.json',
|
|
||||||
'package-lock.json',
|
|
||||||
'web-ext-config.cjs',
|
|
||||||
'web-ext-artifacts',
|
|
||||||
'node_modules',
|
|
||||||
'README.md',
|
|
||||||
'.gitignore',
|
|
||||||
],
|
|
||||||
};
|
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
<RouterView />
|
<RouterView />
|
||||||
</AppShell>
|
</AppShell>
|
||||||
<ImageViewer v-if="modal.isOpen" @close="modal.close()" />
|
<ImageViewer v-if="modal.isOpen" @close="modal.close()" />
|
||||||
|
<PostModal />
|
||||||
<AppSnackbar ref="snackbar" />
|
<AppSnackbar ref="snackbar" />
|
||||||
</v-app>
|
</v-app>
|
||||||
</template>
|
</template>
|
||||||
@@ -13,6 +14,7 @@ import { onMounted, ref } from 'vue'
|
|||||||
import AppShell from './components/AppShell.vue'
|
import AppShell from './components/AppShell.vue'
|
||||||
import AppSnackbar from './components/AppSnackbar.vue'
|
import AppSnackbar from './components/AppSnackbar.vue'
|
||||||
import ImageViewer from './components/modal/ImageViewer.vue'
|
import ImageViewer from './components/modal/ImageViewer.vue'
|
||||||
|
import PostModal from './components/posts/PostModal.vue'
|
||||||
import { useModalStore } from './stores/modal.js'
|
import { useModalStore } from './stores/modal.js'
|
||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-danger-zone mt-8" variant="outlined">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-alert-octagon" color="error" size="small" />
|
||||||
|
<span>Danger zone</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="text-body-2 fc-muted mb-4">
|
||||||
|
Cascade-delete this artist and every image, source, post, and
|
||||||
|
attachment associated with them. This cannot be undone.
|
||||||
|
Recoverable only from an FC-3h backup.
|
||||||
|
</p>
|
||||||
|
<v-btn
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete-forever"
|
||||||
|
:loading="loading"
|
||||||
|
@click="onClick"
|
||||||
|
>Delete artist & cascade</v-btn>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-model="modalOpen"
|
||||||
|
action="delete"
|
||||||
|
kind="artist"
|
||||||
|
:run-id="artistId"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
:description="modalDescription"
|
||||||
|
@confirm="onConfirm"
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
|
import { useRouter } from 'vue-router'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useAdminStore } from '../../stores/admin.js'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
slug: { type: String, required: true },
|
||||||
|
artistId: { type: Number, required: true },
|
||||||
|
artistName: { type: String, required: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
const router = useRouter()
|
||||||
|
const store = useAdminStore()
|
||||||
|
const loading = ref(false)
|
||||||
|
const modalOpen = ref(false)
|
||||||
|
const projected = ref(null)
|
||||||
|
|
||||||
|
const projectedCounts = computed(() => projected.value?.projected || null)
|
||||||
|
|
||||||
|
const modalDescription = computed(
|
||||||
|
() => projected.value
|
||||||
|
? `Artist “${props.artistName}” — `
|
||||||
|
+ `${projected.value.projected.images} images, `
|
||||||
|
+ `${projected.value.projected.sources} sources, `
|
||||||
|
+ `${Math.round(projected.value.projected.bytes_on_disk / 1_048_576)} MiB on disk`
|
||||||
|
: '',
|
||||||
|
)
|
||||||
|
|
||||||
|
async function onClick() {
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
projected.value = await store.projectArtistCascade(props.slug)
|
||||||
|
modalOpen.value = true
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirm(token) {
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
const result = await store.dispatchArtistCascade(props.slug, token)
|
||||||
|
const taskId = result.task_id
|
||||||
|
router.push('/artists')
|
||||||
|
if (taskId) {
|
||||||
|
store.pollTaskUntilDone(taskId).catch(() => {})
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-danger-zone {
|
||||||
|
border-color: rgb(var(--v-theme-error));
|
||||||
|
border-radius: 8px;
|
||||||
|
}
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-artist-gallery">
|
||||||
|
<MasonryGrid
|
||||||
|
:items="store.images"
|
||||||
|
:loading="store.imagesLoading"
|
||||||
|
:has-more="store.hasMoreImages"
|
||||||
|
@load-more="store.loadMoreImages(props.slug)"
|
||||||
|
@open="openImage"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, watch } from 'vue'
|
||||||
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
|
||||||
|
import { useArtistStore } from '../../stores/artist.js'
|
||||||
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
|
import MasonryGrid from '../discovery/MasonryGrid.vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
slug: { type: String, required: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
const store = useArtistStore()
|
||||||
|
const modal = useModalStore()
|
||||||
|
const route = useRoute()
|
||||||
|
const router = useRouter()
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
const initial = parseInt(route.query.image, 10)
|
||||||
|
if (!isNaN(initial)) modal.open(initial)
|
||||||
|
})
|
||||||
|
|
||||||
|
watch(() => route.query.image, (q) => {
|
||||||
|
const id = parseInt(q, 10)
|
||||||
|
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
||||||
|
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
function openImage (id) {
|
||||||
|
router.push({ query: { ...route.query, image: id } })
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-artist-gallery { min-width: 0; }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
<template>
|
||||||
|
<header class="fc-artist-header">
|
||||||
|
<div class="fc-artist-header__left">
|
||||||
|
<h1 class="fc-artist-header__name">{{ name }}</h1>
|
||||||
|
<span v-if="stats" class="fc-artist-header__stats">{{ stats }}</span>
|
||||||
|
</div>
|
||||||
|
<v-tabs
|
||||||
|
:model-value="modelValue"
|
||||||
|
color="accent"
|
||||||
|
density="compact"
|
||||||
|
class="fc-artist-header__tabs"
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)"
|
||||||
|
>
|
||||||
|
<v-tab value="posts">
|
||||||
|
Posts
|
||||||
|
<span v-if="postCount != null" class="fc-artist-header__tab-count">
|
||||||
|
({{ postCount }})
|
||||||
|
</span>
|
||||||
|
</v-tab>
|
||||||
|
<v-tab value="gallery">
|
||||||
|
Gallery
|
||||||
|
<span v-if="imageCount != null" class="fc-artist-header__tab-count">
|
||||||
|
({{ imageCount }})
|
||||||
|
</span>
|
||||||
|
</v-tab>
|
||||||
|
<v-tab value="management">Management</v-tab>
|
||||||
|
</v-tabs>
|
||||||
|
|
||||||
|
<!-- Right-side spacer: balances the left cell's flex weight so the
|
||||||
|
centered tabs stay geometrically centered regardless of the
|
||||||
|
artist-name length. Mirrors TopNav's 1fr | auto | 1fr layout. -->
|
||||||
|
<div class="fc-artist-header__right" />
|
||||||
|
</header>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
name: { type: String, required: true },
|
||||||
|
imageCount: { type: Number, default: null },
|
||||||
|
postCount: { type: Number, default: null },
|
||||||
|
lastAdded: { type: String, default: null },
|
||||||
|
modelValue: { type: String, required: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
defineEmits(['update:modelValue'])
|
||||||
|
|
||||||
|
const stats = computed(() => {
|
||||||
|
const parts = []
|
||||||
|
if (props.imageCount != null) {
|
||||||
|
parts.push(`${props.imageCount} image${props.imageCount === 1 ? '' : 's'}`)
|
||||||
|
}
|
||||||
|
if (props.lastAdded) {
|
||||||
|
parts.push(`last added ${props.lastAdded.slice(0, 10)}`)
|
||||||
|
}
|
||||||
|
return parts.join(' · ')
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
/* Matches TopNav.vue's frosted recipe exactly. top:48px parks it flush
|
||||||
|
against TopNav's bottom edge (TopNav is 0.75rem padding + ~24px content
|
||||||
|
= ~48px tall; operator-flagged 2026-05-26 that top:64px left a visible
|
||||||
|
gap). The two bars now read as one continuous frosted strip. */
|
||||||
|
.fc-artist-header {
|
||||||
|
position: sticky;
|
||||||
|
top: 48px;
|
||||||
|
z-index: 4;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1rem;
|
||||||
|
padding: 0.5rem 1rem;
|
||||||
|
background: linear-gradient(
|
||||||
|
to bottom,
|
||||||
|
rgba(20, 23, 26, 0.92) 0%,
|
||||||
|
rgba(20, 23, 26, 0.65) 60%,
|
||||||
|
rgba(20, 23, 26, 0) 100%
|
||||||
|
);
|
||||||
|
backdrop-filter: blur(2px);
|
||||||
|
-webkit-backdrop-filter: blur(2px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__left {
|
||||||
|
flex: 1 1 0;
|
||||||
|
min-width: 0;
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: 12px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__name {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 24px;
|
||||||
|
font-weight: 500;
|
||||||
|
margin: 0;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
white-space: nowrap;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__stats {
|
||||||
|
font-size: 13px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
white-space: nowrap;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__tabs {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__right {
|
||||||
|
flex: 1 1 0;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-artist-header__tab-count {
|
||||||
|
margin-left: 4px;
|
||||||
|
font-size: 12px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-artist-mgmt">
|
||||||
|
<section class="fc-artist-mgmt__sec">
|
||||||
|
<h2 class="fc-h2">Overview</h2>
|
||||||
|
<div class="fc-artist-mgmt__chips">
|
||||||
|
<v-chip
|
||||||
|
size="small"
|
||||||
|
:variant="overview.is_subscription ? 'flat' : 'outlined'"
|
||||||
|
:color="overview.is_subscription ? 'accent' : undefined"
|
||||||
|
prepend-icon="mdi-rss"
|
||||||
|
>{{ overview.is_subscription ? 'Subscription' : 'One-off' }}</v-chip>
|
||||||
|
<v-chip
|
||||||
|
size="small" variant="outlined" prepend-icon="mdi-link-variant"
|
||||||
|
:to="`/subscriptions?artist_id=${overview.id}`"
|
||||||
|
>{{ overview.sources.length }} subscription{{ overview.sources.length === 1 ? '' : 's' }}</v-chip>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="overview.cooccurring_tags.length" class="fc-artist-mgmt__sec">
|
||||||
|
<h2 class="fc-h2">Frequent tags</h2>
|
||||||
|
<div class="fc-artist-mgmt__tags">
|
||||||
|
<v-chip
|
||||||
|
v-for="t in overview.cooccurring_tags" :key="t.id"
|
||||||
|
size="small" @click="openTag(t.id)"
|
||||||
|
>{{ t.name }} <span class="fc-artist-mgmt__tagc">{{ t.count }}</span></v-chip>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="overview.activity.length" class="fc-artist-mgmt__sec">
|
||||||
|
<h2 class="fc-h2">Activity</h2>
|
||||||
|
<svg
|
||||||
|
class="fc-artist-mgmt__spark" :viewBox="`0 0 ${sparkW} ${sparkH}`"
|
||||||
|
preserveAspectRatio="none" role="img" aria-label="posts over time"
|
||||||
|
>
|
||||||
|
<polyline :points="sparkPoints" fill="none"
|
||||||
|
stroke="rgb(var(--v-theme-accent))" stroke-width="2" />
|
||||||
|
</svg>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="overview.sources.length" class="fc-artist-mgmt__sec">
|
||||||
|
<div class="fc-artist-mgmt__sec-head">
|
||||||
|
<h2 class="fc-h2">Subscriptions</h2>
|
||||||
|
<RouterLink
|
||||||
|
:to="`/subscriptions?artist_id=${overview.id}`"
|
||||||
|
class="fc-artist-mgmt__manage"
|
||||||
|
>Manage subscriptions →</RouterLink>
|
||||||
|
</div>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="s in overview.sources" :key="s.id">
|
||||||
|
<td>{{ s.platform }}</td>
|
||||||
|
<td class="fc-artist-mgmt__url">{{ s.url }}</td>
|
||||||
|
<td class="text-right">{{ s.image_count }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="fc-artist-mgmt__sec">
|
||||||
|
<h2 class="fc-h2">Danger zone</h2>
|
||||||
|
<ArtistDangerZone
|
||||||
|
:slug="overview.slug"
|
||||||
|
:artist-id="overview.id"
|
||||||
|
:artist-name="overview.name"
|
||||||
|
/>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { useRouter, RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import ArtistDangerZone from './ArtistDangerZone.vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
overview: { type: Object, required: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
const router = useRouter()
|
||||||
|
|
||||||
|
const sparkW = 600
|
||||||
|
const sparkH = 80
|
||||||
|
const sparkPoints = computed(() => {
|
||||||
|
const a = props.overview.activity ?? []
|
||||||
|
if (a.length === 0) return ''
|
||||||
|
const max = Math.max(...a.map(p => p.count), 1)
|
||||||
|
const stepX = a.length > 1 ? sparkW / (a.length - 1) : 0
|
||||||
|
return a.map((p, i) => {
|
||||||
|
const x = i * stepX
|
||||||
|
const y = sparkH - (p.count / max) * (sparkH - 4) - 2
|
||||||
|
return `${x.toFixed(1)},${y.toFixed(1)}`
|
||||||
|
}).join(' ')
|
||||||
|
})
|
||||||
|
|
||||||
|
function openTag (tagId) {
|
||||||
|
router.push({ name: 'gallery', query: { tag_id: tagId } })
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-artist-mgmt { padding-top: 1rem; }
|
||||||
|
.fc-h2 {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 20px; font-weight: 500; margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.fc-artist-mgmt__sec { margin-bottom: 28px; }
|
||||||
|
.fc-artist-mgmt__chips { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||||
|
.fc-artist-mgmt__tags { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||||
|
.fc-artist-mgmt__tagc {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
margin-left: 4px;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
.fc-artist-mgmt__spark { width: 100%; height: 80px; }
|
||||||
|
.fc-artist-mgmt__url {
|
||||||
|
max-width: 380px; overflow: hidden; text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-artist-mgmt__sec-head {
|
||||||
|
display: flex; align-items: baseline; justify-content: space-between;
|
||||||
|
margin-bottom: 0.25rem;
|
||||||
|
}
|
||||||
|
.fc-artist-mgmt__manage {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.fc-artist-mgmt__manage:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-artist-posts">
|
||||||
|
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||||
|
{{ String(store.error) }}
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<div v-if="store.loading && store.items.length === 0" class="fc-artist-posts__loading">
|
||||||
|
<v-progress-circular indeterminate color="accent" size="36" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-else-if="store.items.length === 0 && store.done" class="fc-artist-posts__empty">
|
||||||
|
<p>No posts for this artist yet. Switch to
|
||||||
|
<a href="#" @click.prevent="$emit('switch-tab', 'gallery')">Gallery</a>
|
||||||
|
to see imported images, or visit
|
||||||
|
<RouterLink to="/subscriptions">Subscriptions</RouterLink>
|
||||||
|
to start capturing posts.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-else>
|
||||||
|
<PostCard v-for="p in store.items" :key="p.id" :post="p" />
|
||||||
|
<div ref="sentinel" class="fc-artist-posts__sentinel">
|
||||||
|
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
|
||||||
|
<span v-else-if="store.done" class="fc-artist-posts__end">End of stream</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
||||||
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
|
import PostCard from '../posts/PostCard.vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
artistId: { type: Number, required: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
defineEmits(['switch-tab'])
|
||||||
|
|
||||||
|
const store = usePostsStore()
|
||||||
|
const sentinel = ref(null)
|
||||||
|
let observer = null
|
||||||
|
|
||||||
|
async function reload () {
|
||||||
|
await store.loadInitial({ artist_id: props.artistId, platform: null })
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(() => props.artistId, reload)
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await reload()
|
||||||
|
observer = new IntersectionObserver((entries) => {
|
||||||
|
if (entries.some(e => e.isIntersecting)) {
|
||||||
|
store.loadMore()
|
||||||
|
}
|
||||||
|
}, { rootMargin: '400px 0px' })
|
||||||
|
if (sentinel.value) observer.observe(sentinel.value)
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (observer) observer.disconnect()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-artist-posts {
|
||||||
|
max-width: 1600px;
|
||||||
|
margin: 0 auto;
|
||||||
|
}
|
||||||
|
.fc-artist-posts__loading,
|
||||||
|
.fc-artist-posts__empty {
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 2rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-artist-posts__sentinel {
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 1.5rem 0;
|
||||||
|
min-height: 2rem;
|
||||||
|
}
|
||||||
|
.fc-artist-posts__end {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-image-size-select-small" size="small" />
|
||||||
|
<span>Minimum dimensions</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Find and delete images smaller than the threshold. Mirrors the
|
||||||
|
import-time <code>min_width</code> / <code>min_height</code>
|
||||||
|
filter, applied retroactively to the existing library.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-row dense>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="minW" label="Min width (px)" type="number"
|
||||||
|
min="0" density="compact" hide-details
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="minH" label="Min height (px)" type="number"
|
||||||
|
min="0" density="compact" hide-details
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<div class="d-flex align-center mt-3" style="gap: 10px;">
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onPreview"
|
||||||
|
>Preview</v-btn>
|
||||||
|
|
||||||
|
<span v-if="preview" class="text-body-2">
|
||||||
|
<strong>{{ preview.count }}</strong> image(s) would be deleted.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="preview && preview.count > 0"
|
||||||
|
class="mt-3"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onDeleteClick"
|
||||||
|
>Delete {{ preview.count }} matching...</v-btn>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="min-dim"
|
||||||
|
:run-id="tokenSha8"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
:description="`Width < ${minW} OR height < ${minH}`"
|
||||||
|
@confirm="onConfirmedDelete"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const minW = ref(0)
|
||||||
|
const minH = ref(0)
|
||||||
|
const preview = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const tokenSha8 = ref('')
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
minW.value = store.defaults.min_width
|
||||||
|
minH.value = store.defaults.min_height
|
||||||
|
})
|
||||||
|
|
||||||
|
// SHA-256 truncated to 8 hex chars — matches the backend's
|
||||||
|
// _min_dim_token() exactly. Web Crypto rejects MD5 as insecure.
|
||||||
|
async function sha8(canon) {
|
||||||
|
const enc = new TextEncoder()
|
||||||
|
const buf = await crypto.subtle.digest('SHA-256', enc.encode(canon))
|
||||||
|
const hex = Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('')
|
||||||
|
return hex.slice(0, 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPreview() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
preview.value = await store.previewMinDim(minW.value, minH.value)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Preview failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onDeleteClick() {
|
||||||
|
tokenSha8.value = await sha8(`${minW.value}x${minH.value}`)
|
||||||
|
projectedCounts.value = { 'Images to delete': preview.value.count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedDelete(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.deleteMinDim(minW.value, minH.value, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
preview.value = null
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Delete failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-palette-swatch" size="small" />
|
||||||
|
<span>Single-color audit</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Scan library for images dominated by one color within the
|
||||||
|
tolerance. Catches placeholder / solid-fill / error-page images
|
||||||
|
that slipped through the import filter. Same background-scan
|
||||||
|
cadence as the transparency audit.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-row dense>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="threshold" label="Threshold (0–1)"
|
||||||
|
type="number" min="0" max="1" step="0.01"
|
||||||
|
density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="tolerance" label="Color tolerance (0–441)"
|
||||||
|
type="number" min="0" max="441"
|
||||||
|
density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="!audit || audit.status !== 'running'"
|
||||||
|
class="mt-3"
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify-scan"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onStart"
|
||||||
|
>Scan library</v-btn>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||||
|
<v-progress-linear indeterminate color="accent" />
|
||||||
|
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||||
|
<span>
|
||||||
|
Scanning… {{ audit.scanned_count }} checked,
|
||||||
|
{{ audit.matched_count }} matched
|
||||||
|
</span>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" color="warning" rounded="pill"
|
||||||
|
@click="onCancel"
|
||||||
|
>Cancel</v-btn>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||||
|
image(s) match.
|
||||||
|
</p>
|
||||||
|
<v-btn
|
||||||
|
v-if="audit.matched_count > 0"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onApplyClick"
|
||||||
|
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'error'"
|
||||||
|
type="error" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Scan failed: {{ audit.error }}</v-alert>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'applied'"
|
||||||
|
type="success" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Applied — matched images deleted.</v-alert>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="audit"
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="audit"
|
||||||
|
:run-id="audit.id"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
description="Permanently deletes images matched by the single-color scan."
|
||||||
|
@confirm="onConfirmedApply"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const threshold = ref(0.95)
|
||||||
|
const tolerance = ref(30)
|
||||||
|
const audit = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
let pollTimer = null
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
threshold.value = store.defaults.single_color_threshold
|
||||||
|
tolerance.value = store.defaults.single_color_tolerance
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => stopPoll())
|
||||||
|
|
||||||
|
function startPoll(id) {
|
||||||
|
stopPoll()
|
||||||
|
pollTimer = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const fresh = await store.getAudit(id)
|
||||||
|
audit.value = fresh
|
||||||
|
if (fresh.status !== 'running') stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
stopPoll()
|
||||||
|
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}, 5000)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPoll() {
|
||||||
|
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onStart() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
const res = await store.startAudit('single_color', {
|
||||||
|
threshold: threshold.value, tolerance: tolerance.value,
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
|
startPoll(res.audit_id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCancel() {
|
||||||
|
if (!audit.value) return
|
||||||
|
try {
|
||||||
|
await store.cancelAudit(audit.value.id)
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onApplyClick() {
|
||||||
|
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedApply(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-checkerboard" size="small" />
|
||||||
|
<span>Transparency audit</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Scan library for images whose transparent-pixel fraction exceeds
|
||||||
|
the threshold. Animated WebPs / GIFs are skipped (the import-side
|
||||||
|
rule does the same). Runs as a background task — ~50ms per image,
|
||||||
|
so a 57k library takes ~50 minutes.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="threshold" label="Transparency threshold (0–1)"
|
||||||
|
type="number" min="0" max="1" step="0.01" density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
class="mb-3"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="!audit || audit.status !== 'running'"
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify-scan"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onStart"
|
||||||
|
>Scan library</v-btn>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||||
|
<v-progress-linear indeterminate color="accent" />
|
||||||
|
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||||
|
<span>
|
||||||
|
Scanning… {{ audit.scanned_count }} checked,
|
||||||
|
{{ audit.matched_count }} matched
|
||||||
|
</span>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" color="warning" rounded="pill"
|
||||||
|
@click="onCancel"
|
||||||
|
>Cancel</v-btn>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||||
|
image(s) match.
|
||||||
|
</p>
|
||||||
|
<v-btn
|
||||||
|
v-if="audit.matched_count > 0"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onApplyClick"
|
||||||
|
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'error'"
|
||||||
|
type="error" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Scan failed: {{ audit.error }}</v-alert>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'applied'"
|
||||||
|
type="success" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Applied — matched images deleted.</v-alert>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="audit"
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="audit"
|
||||||
|
:run-id="audit.id"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
description="Permanently deletes images matched by the transparency scan."
|
||||||
|
@confirm="onConfirmedApply"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const threshold = ref(0.9)
|
||||||
|
const audit = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
let pollTimer = null
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
threshold.value = store.defaults.transparency_threshold
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => stopPoll())
|
||||||
|
|
||||||
|
function startPoll(id) {
|
||||||
|
stopPoll()
|
||||||
|
pollTimer = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const fresh = await store.getAudit(id)
|
||||||
|
audit.value = fresh
|
||||||
|
if (fresh.status !== 'running') stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
stopPoll()
|
||||||
|
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}, 5000)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPoll() {
|
||||||
|
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onStart() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
const res = await store.startAudit('transparency', { threshold: threshold.value })
|
||||||
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
|
startPoll(res.audit_id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCancel() {
|
||||||
|
if (!audit.value) return
|
||||||
|
try {
|
||||||
|
await store.cancelAudit(audit.value.id)
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onApplyClick() {
|
||||||
|
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedApply(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
<template>
|
||||||
|
<v-dialog :model-value="modelValue" max-width="900"
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 12px;">
|
||||||
|
<v-icon icon="mdi-alert-circle-outline" color="error" />
|
||||||
|
<span>{{ displayTitle }}</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn icon variant="text" size="small" @click="close">
|
||||||
|
<v-icon>mdi-close</v-icon>
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<dl v-if="contextRows.length" class="fc-err-context">
|
||||||
|
<template v-for="(row, idx) in contextRows" :key="idx">
|
||||||
|
<dt>{{ row[0] }}</dt>
|
||||||
|
<dd>{{ row[1] }}</dd>
|
||||||
|
</template>
|
||||||
|
</dl>
|
||||||
|
<pre class="fc-err-pre">{{ displayMessage || '(no error message)' }}</pre>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-btn
|
||||||
|
variant="text" rounded="pill" size="small"
|
||||||
|
:prepend-icon="copied ? 'mdi-check' : 'mdi-content-copy'"
|
||||||
|
@click="onCopy"
|
||||||
|
>{{ copied ? 'Copied' : 'Copy' }}</v-btn>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" rounded="pill" @click="close">Close</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, ref, watch } from 'vue'
|
||||||
|
|
||||||
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
modelValue: { type: Boolean, default: false },
|
||||||
|
// Legacy mode: pass title + message strings. Used by callers whose row
|
||||||
|
// shape lacks structured context (e.g. ImportTaskList where `error` is
|
||||||
|
// a plain string on the import_task row, not a TaskRun).
|
||||||
|
title: { type: String, default: 'Error details' },
|
||||||
|
message: { type: String, default: '' },
|
||||||
|
// Row mode: pass the full TaskRun-shaped dict from /api/system_activity.
|
||||||
|
// When set, displayTitle/displayMessage derive from the row and a context
|
||||||
|
// panel of task_name/queue/target/duration/etc. renders above the error.
|
||||||
|
row: { type: Object, default: null },
|
||||||
|
})
|
||||||
|
|
||||||
|
const emit = defineEmits(['update:modelValue'])
|
||||||
|
|
||||||
|
const copied = ref(false)
|
||||||
|
let copiedTimer = null
|
||||||
|
|
||||||
|
const displayTitle = computed(() => {
|
||||||
|
if (props.row) return props.row.error_type || 'Error details'
|
||||||
|
return props.title
|
||||||
|
})
|
||||||
|
|
||||||
|
const displayMessage = computed(() => {
|
||||||
|
if (props.row) return props.row.error_message || ''
|
||||||
|
return props.message
|
||||||
|
})
|
||||||
|
|
||||||
|
function _shortTaskName (name) {
|
||||||
|
if (!name) return ''
|
||||||
|
const parts = String(name).split('.')
|
||||||
|
return parts[parts.length - 1]
|
||||||
|
}
|
||||||
|
|
||||||
|
function _formatDuration (ms) {
|
||||||
|
if (ms == null) return null
|
||||||
|
if (ms < 1000) return `${ms} ms`
|
||||||
|
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
|
||||||
|
return `${(ms / 60_000).toFixed(1)} min`
|
||||||
|
}
|
||||||
|
|
||||||
|
const contextRows = computed(() => {
|
||||||
|
const r = props.row
|
||||||
|
if (!r) return []
|
||||||
|
const rows = []
|
||||||
|
if (r.task_name) rows.push(['Task', _shortTaskName(r.task_name)])
|
||||||
|
if (r.queue) rows.push(['Queue', r.queue])
|
||||||
|
if (r.target_id != null) rows.push(['Target', r.target_id])
|
||||||
|
const dur = _formatDuration(r.duration_ms)
|
||||||
|
if (dur != null) rows.push(['Duration', dur])
|
||||||
|
if (r.started_at) rows.push(['Started', r.started_at])
|
||||||
|
if (r.finished_at) rows.push(['Finished', r.finished_at])
|
||||||
|
if (r.retry_count) rows.push(['Retries', r.retry_count])
|
||||||
|
if (r.worker_hostname) rows.push(['Worker', r.worker_hostname])
|
||||||
|
if (r.celery_task_id) rows.push(['Celery ID', r.celery_task_id])
|
||||||
|
if (r.args_summary) rows.push(['Args', r.args_summary])
|
||||||
|
return rows
|
||||||
|
})
|
||||||
|
|
||||||
|
watch(() => props.modelValue, (open) => {
|
||||||
|
if (!open) {
|
||||||
|
copied.value = false
|
||||||
|
if (copiedTimer) { clearTimeout(copiedTimer); copiedTimer = null }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
function close () {
|
||||||
|
emit('update:modelValue', false)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCopy () {
|
||||||
|
let text = displayMessage.value || ''
|
||||||
|
if (contextRows.value.length) {
|
||||||
|
const header = contextRows.value.map(([k, v]) => `${k}: ${v}`).join('\n')
|
||||||
|
text = `${header}\n\nError: ${displayTitle.value}\n${text}`
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await copyText(text)
|
||||||
|
copied.value = true
|
||||||
|
if (copiedTimer) clearTimeout(copiedTimer)
|
||||||
|
copiedTimer = setTimeout(() => { copied.value = false }, 1500)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
/* Context panel: muted labels in vellum, crisp values in parchment. The
|
||||||
|
2-column key/value grid keeps rows visually scannable when there are
|
||||||
|
many fields (Task, Queue, Target, Duration, Started, Finished, Retries,
|
||||||
|
Worker, Celery ID, Args). */
|
||||||
|
.fc-err-context {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: max-content 1fr;
|
||||||
|
column-gap: 14px;
|
||||||
|
row-gap: 4px;
|
||||||
|
margin: 0 0 14px 0;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.fc-err-context dt {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
font-weight: 500;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-err-context dd {
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
margin: 0;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Error pre block: high-contrast pairing. The page's `background` token
|
||||||
|
(obsidian #14171A) is darker than the modal card's `surface` (iron
|
||||||
|
#1E2228), so parchment text reads crisply against it. The prior pairing
|
||||||
|
used `surface-variant` which Vuetify auto-derives to a near-parchment
|
||||||
|
light value in this theme — pale-on-pale and unreadable. Operator-
|
||||||
|
flagged 2026-05-26 ("ui contrast is poor"). */
|
||||||
|
.fc-err-pre {
|
||||||
|
font-family: 'JetBrains Mono', ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
word-break: break-word;
|
||||||
|
background: rgb(var(--v-theme-background));
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
padding: 12px 14px;
|
||||||
|
border-radius: 6px;
|
||||||
|
max-height: 60vh;
|
||||||
|
overflow: auto;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -28,6 +28,7 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref } from 'vue'
|
||||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||||
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
|
|
||||||
const store = useCredentialsStore()
|
const store = useCredentialsStore()
|
||||||
const showRotateConfirm = ref(false)
|
const showRotateConfirm = ref(false)
|
||||||
@@ -37,7 +38,7 @@ onMounted(() => store.loadKey())
|
|||||||
async function copyKey() {
|
async function copyKey() {
|
||||||
if (!store.extensionKey) return
|
if (!store.extensionKey) return
|
||||||
try {
|
try {
|
||||||
await navigator.clipboard.writeText(store.extensionKey)
|
await copyText(store.extensionKey)
|
||||||
globalThis.window?.__fcToast?.({ text: 'Copied', type: 'success' })
|
globalThis.window?.__fcToast?.({ text: 'Copied', type: 'success' })
|
||||||
} catch {
|
} catch {
|
||||||
globalThis.window?.__fcToast?.({ text: 'Copy failed', type: 'error' })
|
globalThis.window?.__fcToast?.({ text: 'Copy failed', type: 'error' })
|
||||||
|
|||||||
@@ -38,9 +38,18 @@ function onCardClick() {
|
|||||||
.fc-artistcard { cursor: pointer; }
|
.fc-artistcard { cursor: pointer; }
|
||||||
.fc-artistcard__previews {
|
.fc-artistcard__previews {
|
||||||
display: grid; grid-template-columns: repeat(3, 1fr);
|
display: grid; grid-template-columns: repeat(3, 1fr);
|
||||||
gap: 2px; aspect-ratio: 3 / 1; background: rgb(var(--v-theme-surface-light));
|
gap: 2px; aspect-ratio: 3 / 1;
|
||||||
|
/* Explicit floor + ceiling so tall source images can't escape the
|
||||||
|
preview slot even on browsers where aspect-ratio doesn't compute. */
|
||||||
|
min-height: 150px; max-height: 220px;
|
||||||
|
overflow: hidden;
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
}
|
||||||
|
.fc-artistcard__previews img {
|
||||||
|
display: block;
|
||||||
|
width: 100%; height: 100%;
|
||||||
|
object-fit: cover; object-position: center;
|
||||||
}
|
}
|
||||||
.fc-artistcard__previews img { width: 100%; height: 100%; object-fit: cover; }
|
|
||||||
.fc-artistcard__noimg {
|
.fc-artistcard__noimg {
|
||||||
grid-column: 1 / -1; display: flex; align-items: center;
|
grid-column: 1 / -1; display: flex; align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
|
|||||||
@@ -44,7 +44,31 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="fc-tagcard__meta">
|
<div class="fc-tagcard__meta">
|
||||||
<v-chip size="x-small" label>{{ card.kind }}</v-chip>
|
<v-chip size="x-small" label>{{ card.kind }}</v-chip>
|
||||||
<span class="fc-tagcard__count">{{ card.image_count }}</span>
|
<div class="fc-tagcard__meta-right">
|
||||||
|
<span class="fc-tagcard__count">{{ card.image_count }}</span>
|
||||||
|
<v-menu>
|
||||||
|
<template #activator="{ props: act }">
|
||||||
|
<v-btn
|
||||||
|
class="fc-tagcard__menu"
|
||||||
|
icon="mdi-dots-vertical" size="x-small" variant="text"
|
||||||
|
v-bind="act" @click.stop
|
||||||
|
/>
|
||||||
|
</template>
|
||||||
|
<v-list density="compact">
|
||||||
|
<v-list-item
|
||||||
|
title="Merge with…"
|
||||||
|
prepend-icon="mdi-call-merge"
|
||||||
|
@click="$emit('merge-with', card)"
|
||||||
|
/>
|
||||||
|
<v-list-item
|
||||||
|
title="Delete tag"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
base-color="error"
|
||||||
|
@click="$emit('delete', card)"
|
||||||
|
/>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
@@ -54,7 +78,7 @@
|
|||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
|
|
||||||
const props = defineProps({ card: { type: Object, required: true } })
|
const props = defineProps({ card: { type: Object, required: true } })
|
||||||
const emit = defineEmits(['open', 'rename', 'manage', 'read'])
|
const emit = defineEmits(['open', 'rename', 'manage', 'read', 'merge-with', 'delete'])
|
||||||
|
|
||||||
const editing = ref(false)
|
const editing = ref(false)
|
||||||
const draft = ref('')
|
const draft = ref('')
|
||||||
@@ -82,9 +106,19 @@ function submit() {
|
|||||||
.fc-tagcard { cursor: pointer; }
|
.fc-tagcard { cursor: pointer; }
|
||||||
.fc-tagcard__previews {
|
.fc-tagcard__previews {
|
||||||
display: grid; grid-template-columns: repeat(3, 1fr);
|
display: grid; grid-template-columns: repeat(3, 1fr);
|
||||||
gap: 2px; aspect-ratio: 3 / 1; background: rgb(var(--v-theme-surface-light));
|
gap: 2px; aspect-ratio: 3 / 1;
|
||||||
|
/* Explicit floor + ceiling so tall source images can't escape the
|
||||||
|
preview slot even on browsers where aspect-ratio doesn't compute
|
||||||
|
(older Safari, embedded webviews). */
|
||||||
|
min-height: 150px; max-height: 220px;
|
||||||
|
overflow: hidden;
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
}
|
||||||
|
.fc-tagcard__previews img {
|
||||||
|
display: block;
|
||||||
|
width: 100%; height: 100%;
|
||||||
|
object-fit: cover; object-position: center;
|
||||||
}
|
}
|
||||||
.fc-tagcard__previews img { width: 100%; height: 100%; object-fit: cover; }
|
|
||||||
.fc-tagcard__noimg {
|
.fc-tagcard__noimg {
|
||||||
grid-column: 1 / -1; display: flex; align-items: center;
|
grid-column: 1 / -1; display: flex; align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
@@ -106,4 +140,13 @@ function submit() {
|
|||||||
}
|
}
|
||||||
.fc-tagcard:hover .fc-tagcard__edit { opacity: .6; }
|
.fc-tagcard:hover .fc-tagcard__edit { opacity: .6; }
|
||||||
.fc-tagcard__edit:hover { opacity: 1; }
|
.fc-tagcard__edit:hover { opacity: 1; }
|
||||||
|
.fc-tagcard__meta-right {
|
||||||
|
display: flex; align-items: center; gap: 4px;
|
||||||
|
}
|
||||||
|
.fc-tagcard__menu {
|
||||||
|
opacity: 0;
|
||||||
|
transition: opacity .15s ease;
|
||||||
|
}
|
||||||
|
.fc-tagcard:hover .fc-tagcard__menu { opacity: .6; }
|
||||||
|
.fc-tagcard__menu:hover { opacity: 1; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -55,19 +55,44 @@
|
|||||||
</template>
|
</template>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="fc-bulk-panel__section">
|
||||||
|
<h4>Destructive</h4>
|
||||||
|
<v-btn
|
||||||
|
color="error" variant="flat" rounded="pill" block
|
||||||
|
prepend-icon="mdi-delete-forever"
|
||||||
|
:disabled="!sel.count"
|
||||||
|
:loading="deleting"
|
||||||
|
@click="onDeleteClick"
|
||||||
|
>Delete {{ sel.count }} selected</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="fc-bulk-panel__foot">
|
<div class="fc-bulk-panel__foot">
|
||||||
<v-btn variant="text" block @click="sel.clear()">Clear selection</v-btn>
|
<v-btn variant="text" block @click="sel.clear()">Clear selection</v-btn>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-model="deleteModalOpen"
|
||||||
|
action="delete"
|
||||||
|
kind="images-selection"
|
||||||
|
:run-id="bulkToken"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="bulkProjectedCounts"
|
||||||
|
:description="bulkDescription"
|
||||||
|
@confirm="onDeleteConfirm"
|
||||||
|
/>
|
||||||
</aside>
|
</aside>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { ref, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
import { useGallerySelectionStore } from '../../stores/gallerySelection.js'
|
import { useGallerySelectionStore } from '../../stores/gallerySelection.js'
|
||||||
|
import { useAdminStore } from '../../stores/admin.js'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
import { useApi } from '../../composables/useApi.js'
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
|
||||||
const sel = useGallerySelectionStore()
|
const sel = useGallerySelectionStore()
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
|
const adminStore = useAdminStore()
|
||||||
|
|
||||||
const addModel = ref(null)
|
const addModel = ref(null)
|
||||||
const addHits = ref([])
|
const addHits = ref([])
|
||||||
@@ -99,6 +124,66 @@ async function onAddPick(id) {
|
|||||||
watch(() => sel.order.length, () => {
|
watch(() => sel.order.length, () => {
|
||||||
if (sel.isSelectMode) sel.refresh()
|
if (sel.isSelectMode) sel.refresh()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// --- FC-3k bulk delete -----------------------------------------------
|
||||||
|
|
||||||
|
const deleting = ref(false)
|
||||||
|
const deleteModalOpen = ref(false)
|
||||||
|
const bulkProjected = ref(null)
|
||||||
|
const bulkToken = ref('')
|
||||||
|
|
||||||
|
const bulkProjectedCounts = computed(() => bulkProjected.value
|
||||||
|
? {
|
||||||
|
images: bulkProjected.value.images_found,
|
||||||
|
thumbnails: bulkProjected.value.thumbs_to_unlink,
|
||||||
|
bytes: bulkProjected.value.bytes_on_disk,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
)
|
||||||
|
const bulkDescription = computed(
|
||||||
|
() => bulkProjected.value
|
||||||
|
? `${bulkProjected.value.images_found} images, `
|
||||||
|
+ `${Math.round(bulkProjected.value.bytes_on_disk / 1_048_576)} MiB on disk`
|
||||||
|
: '',
|
||||||
|
)
|
||||||
|
|
||||||
|
async function _computeSha8(ids) {
|
||||||
|
const canon = [...ids].sort((a, b) => a - b).join(',')
|
||||||
|
const buf = new TextEncoder().encode(canon)
|
||||||
|
const hashBuf = await crypto.subtle.digest('SHA-256', buf)
|
||||||
|
const bytes = new Uint8Array(hashBuf)
|
||||||
|
let hex = ''
|
||||||
|
for (let i = 0; i < 4; i++) {
|
||||||
|
hex += bytes[i].toString(16).padStart(2, '0')
|
||||||
|
}
|
||||||
|
return hex
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onDeleteClick() {
|
||||||
|
if (!sel.order.length) return
|
||||||
|
deleting.value = true
|
||||||
|
try {
|
||||||
|
bulkProjected.value = await adminStore.projectBulkImageDelete(sel.order)
|
||||||
|
bulkToken.value = await _computeSha8(sel.order)
|
||||||
|
deleteModalOpen.value = true
|
||||||
|
} finally {
|
||||||
|
deleting.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onDeleteConfirm(token) {
|
||||||
|
deleting.value = true
|
||||||
|
try {
|
||||||
|
const result = await adminStore.dispatchBulkImageDelete(sel.order, token)
|
||||||
|
const taskId = result.task_id
|
||||||
|
if (taskId) {
|
||||||
|
adminStore.pollTaskUntilDone(taskId).catch(() => {})
|
||||||
|
}
|
||||||
|
sel.clear()
|
||||||
|
} finally {
|
||||||
|
deleting.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
<template>
|
||||||
|
<v-dialog
|
||||||
|
:model-value="modelValue"
|
||||||
|
max-width="520" persistent
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)"
|
||||||
|
>
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>
|
||||||
|
{{ titleVerb }}
|
||||||
|
{{ kindLabel }}<span v-if="runId"> #{{ runId }}</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<v-alert
|
||||||
|
:type="alertType"
|
||||||
|
variant="tonal" density="compact" class="mb-3"
|
||||||
|
>
|
||||||
|
<strong>{{ warningText }}</strong>
|
||||||
|
<div v-if="description" class="mt-1">{{ description }}</div>
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<div v-if="projectedCounts" class="fc-counts mb-3">
|
||||||
|
<div v-for="(v, k) in projectedCounts" :key="k">
|
||||||
|
<span class="fc-counts-key">{{ k }}:</span>
|
||||||
|
<span class="fc-counts-val">{{ v }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<template v-if="tier === 'C'">
|
||||||
|
<div class="text-body-2 mb-2">Type the following to confirm:</div>
|
||||||
|
<div class="fc-token mb-3">{{ expectedToken }}</div>
|
||||||
|
<v-text-field
|
||||||
|
v-model="typed"
|
||||||
|
variant="outlined" density="compact" hide-details
|
||||||
|
autofocus
|
||||||
|
placeholder="paste the token above"
|
||||||
|
/>
|
||||||
|
</template>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="onCancel">Cancel</v-btn>
|
||||||
|
<v-btn
|
||||||
|
:color="confirmColor"
|
||||||
|
variant="flat" rounded="pill"
|
||||||
|
:disabled="!canConfirm"
|
||||||
|
@click="onConfirm"
|
||||||
|
>
|
||||||
|
{{ titleVerb }}
|
||||||
|
</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, ref, watch } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
modelValue: { type: Boolean, required: true },
|
||||||
|
action: { type: String, required: true }, // 'restore' | 'delete'
|
||||||
|
kind: { type: String, required: true }, // 'db' | 'images' | 'artist' | 'tag' | 'images-selection'
|
||||||
|
runId: { type: [Number, String], default: '' }, // numeric id or sha8 string
|
||||||
|
description: { type: String, default: '' },
|
||||||
|
tier: { type: String, default: 'C' }, // 'B' | 'C'
|
||||||
|
projectedCounts: { type: Object, default: null },
|
||||||
|
})
|
||||||
|
const emit = defineEmits(['update:modelValue', 'confirm'])
|
||||||
|
|
||||||
|
const typed = ref('')
|
||||||
|
const expectedToken = computed(
|
||||||
|
() => `${props.action}-${props.kind}-${props.runId}`,
|
||||||
|
)
|
||||||
|
const titleVerb = computed(
|
||||||
|
() => props.action === 'restore' ? 'Restore' : 'Delete',
|
||||||
|
)
|
||||||
|
const kindLabel = computed(() => ({
|
||||||
|
db: 'database backup',
|
||||||
|
images: 'images backup',
|
||||||
|
artist: 'artist',
|
||||||
|
tag: 'tag',
|
||||||
|
'images-selection': 'image selection',
|
||||||
|
}[props.kind] || props.kind))
|
||||||
|
const alertType = computed(
|
||||||
|
() => props.action === 'restore' ? 'warning' : 'error',
|
||||||
|
)
|
||||||
|
const confirmColor = computed(
|
||||||
|
() => props.action === 'restore' ? 'warning' : 'error',
|
||||||
|
)
|
||||||
|
const warningText = computed(() => (
|
||||||
|
props.action === 'restore'
|
||||||
|
? 'This replaces current state with the backup. There is no undo.'
|
||||||
|
: 'This permanently deletes the listed items. Cannot be recovered.'
|
||||||
|
))
|
||||||
|
const canConfirm = computed(
|
||||||
|
() => props.tier === 'B'
|
||||||
|
? true
|
||||||
|
: typed.value === expectedToken.value,
|
||||||
|
)
|
||||||
|
|
||||||
|
watch(() => props.modelValue, (open) => {
|
||||||
|
if (open) typed.value = ''
|
||||||
|
})
|
||||||
|
|
||||||
|
function onCancel() {
|
||||||
|
emit('update:modelValue', false)
|
||||||
|
}
|
||||||
|
function onConfirm() {
|
||||||
|
emit('confirm', expectedToken.value)
|
||||||
|
emit('update:modelValue', false)
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-token {
|
||||||
|
font-family: 'JetBrains Mono', monospace;
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
padding: 6px 10px; border-radius: 4px;
|
||||||
|
font-size: 14px; word-break: break-all;
|
||||||
|
}
|
||||||
|
.fc-counts {
|
||||||
|
display: grid; grid-template-columns: max-content auto;
|
||||||
|
gap: 4px 12px;
|
||||||
|
font-size: 13px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-counts-key { font-weight: 500; text-transform: capitalize; }
|
||||||
|
.fc-counts-val { font-variant-numeric: tabular-nums; }
|
||||||
|
</style>
|
||||||
@@ -2,9 +2,6 @@
|
|||||||
<Teleport to="body">
|
<Teleport to="body">
|
||||||
<div
|
<div
|
||||||
class="fc-viewer" role="dialog" aria-modal="true"
|
class="fc-viewer" role="dialog" aria-modal="true"
|
||||||
@keydown.esc="$emit('close')"
|
|
||||||
@keydown.left="onArrowLeft"
|
|
||||||
@keydown.right="onArrowRight"
|
|
||||||
tabindex="-1" ref="rootEl"
|
tabindex="-1" ref="rootEl"
|
||||||
>
|
>
|
||||||
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
|
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
|
||||||
@@ -68,7 +65,7 @@ import VideoCanvas from './VideoCanvas.vue'
|
|||||||
import TagPanel from './TagPanel.vue'
|
import TagPanel from './TagPanel.vue'
|
||||||
import ProvenancePanel from './ProvenancePanel.vue'
|
import ProvenancePanel from './ProvenancePanel.vue'
|
||||||
|
|
||||||
defineEmits(['close'])
|
const emit = defineEmits(['close'])
|
||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const rootEl = ref(null)
|
const rootEl = ref(null)
|
||||||
@@ -86,14 +83,39 @@ const integrityBadge = computed(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
let prevBodyOverflow = null
|
let prevBodyOverflow = null
|
||||||
|
|
||||||
|
// Document-level keyboard handler. Bound on document (not on the modal
|
||||||
|
// root) because <video> elements capture focus when the user interacts
|
||||||
|
// with their controls — when video has focus, Escape and arrow keys
|
||||||
|
// don't bubble reliably to ancestors. Listening on document side-steps
|
||||||
|
// that. Filtered via isTextEntry so tag/comment inputs still get their
|
||||||
|
// own keystrokes.
|
||||||
|
function onKeyDown(ev) {
|
||||||
|
if (ev.key === 'Escape') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
emit('close')
|
||||||
|
} else if (ev.key === 'ArrowLeft') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
modal.goPrev()
|
||||||
|
} else if (ev.key === 'ArrowRight') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
modal.goNext()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
prevBodyOverflow = document.body.style.overflow
|
prevBodyOverflow = document.body.style.overflow
|
||||||
document.body.style.overflow = 'hidden'
|
document.body.style.overflow = 'hidden'
|
||||||
|
document.addEventListener('keydown', onKeyDown, true)
|
||||||
await nextFrame()
|
await nextFrame()
|
||||||
rootEl.value?.focus()
|
rootEl.value?.focus()
|
||||||
})
|
})
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
document.body.style.overflow = prevBodyOverflow
|
document.body.style.overflow = prevBodyOverflow
|
||||||
|
document.removeEventListener('keydown', onKeyDown, true)
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => modal.currentImageId, async () => {
|
watch(() => modal.currentImageId, async () => {
|
||||||
@@ -105,16 +127,6 @@ function nextFrame() {
|
|||||||
return new Promise(resolve => requestAnimationFrame(resolve))
|
return new Promise(resolve => requestAnimationFrame(resolve))
|
||||||
}
|
}
|
||||||
|
|
||||||
function onArrowLeft(ev) {
|
|
||||||
if (isTextEntry(ev.target)) return
|
|
||||||
ev.preventDefault()
|
|
||||||
modal.goPrev()
|
|
||||||
}
|
|
||||||
function onArrowRight(ev) {
|
|
||||||
if (isTextEntry(ev.target)) return
|
|
||||||
ev.preventDefault()
|
|
||||||
modal.goNext()
|
|
||||||
}
|
|
||||||
function isTextEntry(el) {
|
function isTextEntry(el) {
|
||||||
if (!el) return false
|
if (!el) return false
|
||||||
const tag = el.tagName
|
const tag = el.tagName
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-tag-autocomplete">
|
<div class="fc-tag-autocomplete">
|
||||||
<div class="d-flex" style="gap: 6px;">
|
<v-text-field
|
||||||
<v-select
|
v-model="query"
|
||||||
v-model="kind" :items="kindOptions" :item-title="(k) => k.label" :item-value="(k) => k.value"
|
placeholder="Add tag (or kind:name — character/fandom/series)"
|
||||||
density="compact" hide-details style="max-width: 140px;"
|
density="compact" hide-details
|
||||||
/>
|
@keydown.down.prevent="moveHighlight(1)"
|
||||||
<v-text-field
|
@keydown.up.prevent="moveHighlight(-1)"
|
||||||
v-model="query" placeholder="Add tag…" density="compact" hide-details
|
@keydown.enter.prevent="onEnter"
|
||||||
@keydown.down.prevent="moveHighlight(1)" @keydown.up.prevent="moveHighlight(-1)"
|
@keydown.esc="$emit('cancel')"
|
||||||
@keydown.enter.prevent="onEnter" @keydown.esc="$emit('cancel')"
|
/>
|
||||||
/>
|
<v-list
|
||||||
</div>
|
v-if="hits.length || allowCreate"
|
||||||
<v-list v-if="hits.length || allowCreate" density="compact" class="fc-tag-autocomplete__list">
|
density="compact" class="fc-tag-autocomplete__list"
|
||||||
|
>
|
||||||
<v-list-item
|
<v-list-item
|
||||||
v-for="(h, idx) in hits" :key="h.id"
|
v-for="(h, idx) in hits" :key="h.id"
|
||||||
:active="idx === highlight" @click="onPick(h)"
|
:active="idx === highlight" @click="onPick(h)"
|
||||||
@@ -26,7 +27,7 @@
|
|||||||
<span v-if="h.fandom_name" class="text-caption">— {{ h.fandom_name }}</span>
|
<span v-if="h.fandom_name" class="text-caption">— {{ h.fandom_name }}</span>
|
||||||
</v-list-item-title>
|
</v-list-item-title>
|
||||||
<template #append>
|
<template #append>
|
||||||
<span class="text-caption">{{ h.image_count }}</span>
|
<span class="text-caption">{{ h.kind }}</span>
|
||||||
</template>
|
</template>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
<v-list-item
|
<v-list-item
|
||||||
@@ -34,9 +35,13 @@
|
|||||||
@click="onCreate"
|
@click="onCreate"
|
||||||
>
|
>
|
||||||
<template #prepend>
|
<template #prepend>
|
||||||
<v-icon size="small" :color="store.colorFor(kind)">{{ iconFor(kind) }}</v-icon>
|
<v-icon size="small" :color="store.colorFor(parsedKind)">
|
||||||
|
{{ iconFor(parsedKind) }}
|
||||||
|
</v-icon>
|
||||||
</template>
|
</template>
|
||||||
<v-list-item-title>Create "{{ query }}" as {{ kind }}</v-list-item-title>
|
<v-list-item-title>
|
||||||
|
Create "{{ parsedName }}" as {{ parsedKind }}
|
||||||
|
</v-list-item-title>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
</v-list>
|
</v-list>
|
||||||
|
|
||||||
@@ -54,65 +59,95 @@ import FandomPicker from './FandomPicker.vue'
|
|||||||
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
||||||
const store = useTagStore()
|
const store = useTagStore()
|
||||||
|
|
||||||
const kind = ref('general')
|
// Single text input; no kind dropdown. Client-side mirror of the
|
||||||
|
// backend's parse_kind_prefix lives below — kept in sync with
|
||||||
|
// KNOWN_KINDS in backend/app/utils/tag_prefix.py. The backend is the
|
||||||
|
// canonical parser; this mirror just powers the live Create-label
|
||||||
|
// preview ("Create 'Eric' as artist") and the autocomplete query.
|
||||||
const query = ref('')
|
const query = ref('')
|
||||||
const hits = ref([])
|
const hits = ref([])
|
||||||
const highlight = ref(0)
|
const highlight = ref(0)
|
||||||
const fandomDialog = ref(false)
|
const fandomDialog = ref(false)
|
||||||
let pendingNewName = null
|
let pendingNewName = null
|
||||||
|
|
||||||
const kindOptions = store.kindOptions()
|
const KNOWN_KINDS = new Set([
|
||||||
|
'character', 'fandom', 'series',
|
||||||
|
])
|
||||||
const KIND_ICONS = {
|
const KIND_ICONS = {
|
||||||
general: 'mdi-tag', artist: 'mdi-palette', character: 'mdi-account-circle',
|
general: 'mdi-tag', character: 'mdi-account-circle',
|
||||||
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
|
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
|
||||||
meta: 'mdi-cog-outline', rating: 'mdi-shield-check-outline'
|
|
||||||
}
|
}
|
||||||
function iconFor(k) { return KIND_ICONS[k] || 'mdi-tag' }
|
function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
||||||
|
|
||||||
|
const parsed = computed(() => {
|
||||||
|
const raw = query.value.trim()
|
||||||
|
if (raw.includes(':')) {
|
||||||
|
const idx = raw.indexOf(':')
|
||||||
|
const prefix = raw.slice(0, idx).toLowerCase()
|
||||||
|
if (KNOWN_KINDS.has(prefix)) {
|
||||||
|
return { kind: prefix, name: raw.slice(idx + 1).trim() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { kind: 'general', name: raw }
|
||||||
|
})
|
||||||
|
const parsedKind = computed(() => parsed.value.kind)
|
||||||
|
const parsedName = computed(() => parsed.value.name)
|
||||||
|
|
||||||
let debounceId = null
|
let debounceId = null
|
||||||
watch([query, kind], () => {
|
watch(query, () => {
|
||||||
highlight.value = 0
|
highlight.value = 0
|
||||||
if (debounceId) clearTimeout(debounceId)
|
if (debounceId) clearTimeout(debounceId)
|
||||||
debounceId = setTimeout(async () => {
|
debounceId = setTimeout(async () => {
|
||||||
const q = query.value.trim()
|
const q = parsedName.value
|
||||||
if (!q) { hits.value = []; return }
|
if (!q) { hits.value = []; return }
|
||||||
hits.value = await store.autocomplete(q, kind.value, 10)
|
// Autocomplete across ALL kinds. When the user typed a prefix the
|
||||||
|
// matches list is naturally narrower because the parsed name is
|
||||||
|
// shorter; we don't filter server-side by kind.
|
||||||
|
hits.value = await store.autocomplete(q, null, 10)
|
||||||
}, 200)
|
}, 200)
|
||||||
})
|
})
|
||||||
|
|
||||||
const allowCreate = computed(() => {
|
const allowCreate = computed(() => {
|
||||||
const q = query.value.trim()
|
const q = parsedName.value
|
||||||
return q && !hits.value.some(h => h.name.toLowerCase() === q.toLowerCase() && h.kind === kind.value)
|
if (!q) return false
|
||||||
|
return !hits.value.some(h =>
|
||||||
|
h.name.toLowerCase() === q.toLowerCase() && h.kind === parsedKind.value,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
function moveHighlight(delta) {
|
function moveHighlight (delta) {
|
||||||
const total = hits.value.length + (allowCreate.value ? 1 : 0)
|
const total = hits.value.length + (allowCreate.value ? 1 : 0)
|
||||||
if (total === 0) return
|
if (total === 0) return
|
||||||
highlight.value = (highlight.value + delta + total) % total
|
highlight.value = (highlight.value + delta + total) % total
|
||||||
}
|
}
|
||||||
|
|
||||||
function onPick(hit) { emit('pick-existing', hit); reset() }
|
function onPick (hit) { emit('pick-existing', hit); reset() }
|
||||||
|
|
||||||
function onCreate() {
|
function onCreate () {
|
||||||
const name = query.value.trim()
|
const name = parsedName.value
|
||||||
if (kind.value === 'character') {
|
const kind = parsedKind.value
|
||||||
// Character requires a fandom — open the picker.
|
if (kind === 'character') {
|
||||||
pendingNewName = name
|
pendingNewName = name
|
||||||
fandomDialog.value = true
|
fandomDialog.value = true
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
emit('pick-new', { name, kind: kind.value, fandom_id: null })
|
// Pass explicit kind here; the backend accepts both shapes. Passing
|
||||||
|
// it makes the parsed kind preview match the actual server outcome
|
||||||
|
// for users who didn't use a prefix (general goes through cleanly).
|
||||||
|
emit('pick-new', { name, kind, fandom_id: null })
|
||||||
reset()
|
reset()
|
||||||
}
|
}
|
||||||
|
|
||||||
function onFandomChosen(fandom) {
|
function onFandomChosen (fandom) {
|
||||||
fandomDialog.value = false
|
fandomDialog.value = false
|
||||||
emit('pick-new', { name: pendingNewName, kind: 'character', fandom_id: fandom.id })
|
emit('pick-new', {
|
||||||
|
name: pendingNewName, kind: 'character', fandom_id: fandom.id,
|
||||||
|
})
|
||||||
pendingNewName = null
|
pendingNewName = null
|
||||||
reset()
|
reset()
|
||||||
}
|
}
|
||||||
|
|
||||||
function onEnter() {
|
function onEnter () {
|
||||||
if (highlight.value < hits.value.length) {
|
if (highlight.value < hits.value.length) {
|
||||||
onPick(hits.value[highlight.value])
|
onPick(hits.value[highlight.value])
|
||||||
} else if (allowCreate.value) {
|
} else if (allowCreate.value) {
|
||||||
@@ -120,7 +155,7 @@ function onEnter() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function reset() { query.value = ''; hits.value = []; highlight.value = 0 }
|
function reset () { query.value = ''; hits.value = []; highlight.value = 0 }
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -1,10 +1,17 @@
|
|||||||
<template>
|
<template>
|
||||||
<v-card class="fc-post-card" variant="outlined">
|
<v-card
|
||||||
|
class="fc-post-card"
|
||||||
|
variant="outlined"
|
||||||
|
tabindex="0"
|
||||||
|
@click="onCardClick"
|
||||||
|
@keydown.enter="onCardClick"
|
||||||
|
>
|
||||||
<div class="fc-post-card__head">
|
<div class="fc-post-card__head">
|
||||||
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
||||||
<RouterLink
|
<RouterLink
|
||||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||||
class="fc-post-card__artist"
|
class="fc-post-card__artist"
|
||||||
|
@click.stop
|
||||||
>{{ post.artist.name }}</RouterLink>
|
>{{ post.artist.name }}</RouterLink>
|
||||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
@@ -13,72 +20,74 @@
|
|||||||
:href="post.post_url" target="_blank" rel="noopener"
|
:href="post.post_url" target="_blank" rel="noopener"
|
||||||
icon="mdi-open-in-new" size="x-small" variant="text"
|
icon="mdi-open-in-new" size="x-small" variant="text"
|
||||||
:aria-label="`open original post on ${post.source.platform}`"
|
:aria-label="`open original post on ${post.source.platform}`"
|
||||||
|
@click.stop
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h3 v-if="post.post_title" class="fc-post-card__title">{{ post.post_title }}</h3>
|
<div class="fc-post-card__body">
|
||||||
|
<div class="fc-post-card__media">
|
||||||
|
<template v-if="post.thumbnails?.length">
|
||||||
|
<div class="fc-post-card__hero">
|
||||||
|
<img :src="hero.thumbnail_url" :alt="`hero thumbnail`" loading="lazy" />
|
||||||
|
</div>
|
||||||
|
<div v-if="rail.length" class="fc-post-card__rail">
|
||||||
|
<div v-for="t in rail" :key="t.image_id" class="fc-post-card__rail-cell">
|
||||||
|
<img :src="t.thumbnail_url" :alt="`thumbnail`" loading="lazy" />
|
||||||
|
</div>
|
||||||
|
<div v-if="moreCount > 0" class="fc-post-card__rail-more">
|
||||||
|
+{{ moreCount }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
<PostEmptyThumbs v-else />
|
||||||
|
</div>
|
||||||
|
|
||||||
<div v-if="descriptionToShow" class="fc-post-card__desc">
|
<div class="fc-post-card__text">
|
||||||
<span class="fc-post-card__desc-text">{{ descriptionToShow }}</span>
|
<h3 v-if="post.post_title" class="fc-post-card__title">
|
||||||
<button
|
{{ post.post_title }}
|
||||||
v-if="post.description_truncated && !expanded"
|
</h3>
|
||||||
class="fc-post-card__more" type="button"
|
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
||||||
@click="expand"
|
Post {{ post.external_post_id }}
|
||||||
>Show more</button>
|
</h3>
|
||||||
<button
|
|
||||||
v-if="expanded"
|
|
||||||
class="fc-post-card__more" type="button"
|
|
||||||
@click="expanded = false"
|
|
||||||
>Show less</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-if="post.thumbnails.length" class="fc-post-card__thumbs">
|
<p v-if="post.description_plain" class="fc-post-card__desc">
|
||||||
<RouterLink
|
{{ post.description_plain }}
|
||||||
v-for="t in post.thumbnails"
|
</p>
|
||||||
:key="t.image_id"
|
<p v-else class="fc-post-card__desc fc-post-card__desc--missing">
|
||||||
:to="{ path: '/gallery', query: { post_id: post.id } }"
|
(no description)
|
||||||
class="fc-post-card__thumb"
|
</p>
|
||||||
>
|
|
||||||
<v-img
|
|
||||||
:src="t.thumbnail_url" cover :alt="`thumbnail`"
|
|
||||||
width="96" height="96" class="fc-post-card__thumb-img"
|
|
||||||
/>
|
|
||||||
</RouterLink>
|
|
||||||
<RouterLink
|
|
||||||
v-if="post.thumbnails_more > 0"
|
|
||||||
:to="{ path: '/gallery', query: { post_id: post.id } }"
|
|
||||||
class="fc-post-card__more-thumbs"
|
|
||||||
>+{{ post.thumbnails_more }} more</RouterLink>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-if="post.attachments.length" class="fc-post-card__attachments">
|
<div v-if="post.attachments?.length" class="fc-post-card__atts">
|
||||||
<a
|
<v-icon size="small" class="fc-post-card__att-icon">mdi-paperclip</v-icon>
|
||||||
v-for="att in post.attachments"
|
{{ post.attachments.length }} attachment{{ post.attachments.length === 1 ? '' : 's' }}
|
||||||
:key="att.id"
|
</div>
|
||||||
:href="att.download_url"
|
</div>
|
||||||
download
|
|
||||||
class="fc-post-card__att"
|
|
||||||
>
|
|
||||||
<v-icon size="small" class="fc-post-card__att-icon">mdi-paperclip</v-icon>
|
|
||||||
<span class="fc-post-card__att-name">{{ att.original_filename }}</span>
|
|
||||||
<span class="fc-post-card__att-size">({{ formatBytes(att.size_bytes) }})</span>
|
|
||||||
</a>
|
|
||||||
</div>
|
</div>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, ref } from 'vue'
|
import { computed } from 'vue'
|
||||||
import { usePostsStore } from '../../stores/posts.js'
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import { usePostModalStore } from '../../stores/postModal.js'
|
||||||
|
import PostEmptyThumbs from './PostEmptyThumbs.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
post: { type: Object, required: true },
|
post: { type: Object, required: true },
|
||||||
})
|
})
|
||||||
|
|
||||||
const postsStore = usePostsStore()
|
const postModal = usePostModalStore()
|
||||||
|
|
||||||
const expanded = ref(false)
|
const hero = computed(() => props.post.thumbnails?.[0])
|
||||||
const fullDescription = ref(null)
|
const rail = computed(() => (props.post.thumbnails || []).slice(1, 4))
|
||||||
|
const moreCount = computed(() => {
|
||||||
|
const more = props.post.thumbnails_more || 0
|
||||||
|
const railLen = rail.value.length
|
||||||
|
// If feed returned more than (1 hero + 3 rail = 4), extra spills into "+N".
|
||||||
|
const extraShown = Math.max(0, (props.post.thumbnails?.length || 0) - 1 - railLen)
|
||||||
|
return more + extraShown
|
||||||
|
})
|
||||||
|
|
||||||
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
||||||
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
||||||
@@ -92,25 +101,8 @@ const relativeDate = computed(() => {
|
|||||||
return new Date(sortDateIso.value).toLocaleDateString()
|
return new Date(sortDateIso.value).toLocaleDateString()
|
||||||
})
|
})
|
||||||
|
|
||||||
const descriptionToShow = computed(() => {
|
function onCardClick () {
|
||||||
if (expanded.value && fullDescription.value) return fullDescription.value
|
postModal.open(props.post)
|
||||||
return props.post.description_plain
|
|
||||||
})
|
|
||||||
|
|
||||||
async function expand() {
|
|
||||||
if (!fullDescription.value) {
|
|
||||||
const detail = await postsStore.getPostFull(props.post.id)
|
|
||||||
fullDescription.value = detail?.description_full || props.post.description_plain
|
|
||||||
}
|
|
||||||
expanded.value = true
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatBytes(n) {
|
|
||||||
if (!n) return '0 B'
|
|
||||||
if (n < 1024) return `${n} B`
|
|
||||||
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
|
|
||||||
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`
|
|
||||||
return `${(n / 1024 / 1024 / 1024).toFixed(1)} GB`
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@@ -118,13 +110,25 @@ function formatBytes(n) {
|
|||||||
.fc-post-card {
|
.fc-post-card {
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
margin-bottom: 1rem;
|
margin-bottom: 1rem;
|
||||||
|
cursor: pointer;
|
||||||
|
container-type: inline-size;
|
||||||
|
transition: border-color 0.15s ease;
|
||||||
}
|
}
|
||||||
|
.fc-post-card:hover {
|
||||||
|
border-color: rgb(var(--v-theme-accent));
|
||||||
|
}
|
||||||
|
.fc-post-card:focus-visible {
|
||||||
|
outline: 2px solid rgb(var(--v-theme-accent));
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
.fc-post-card__head {
|
.fc-post-card__head {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 0.6rem;
|
gap: 0.6rem;
|
||||||
font-size: 0.85rem;
|
font-size: 0.8rem;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
margin-bottom: 12px;
|
||||||
}
|
}
|
||||||
.fc-post-card__artist {
|
.fc-post-card__artist {
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
@@ -133,72 +137,118 @@ function formatBytes(n) {
|
|||||||
}
|
}
|
||||||
.fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); }
|
.fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
.fc-post-card__date { white-space: nowrap; }
|
.fc-post-card__date { white-space: nowrap; }
|
||||||
.fc-post-card__title {
|
|
||||||
font-size: 1.05rem;
|
.fc-post-card__body {
|
||||||
margin: 0.6rem 0 0.4rem;
|
|
||||||
}
|
|
||||||
.fc-post-card__desc {
|
|
||||||
white-space: pre-wrap;
|
|
||||||
font-size: 0.92rem;
|
|
||||||
color: rgb(var(--v-theme-on-surface));
|
|
||||||
margin-bottom: 0.6rem;
|
|
||||||
}
|
|
||||||
.fc-post-card__more {
|
|
||||||
background: none;
|
|
||||||
border: none;
|
|
||||||
color: rgb(var(--v-theme-accent));
|
|
||||||
cursor: pointer;
|
|
||||||
padding: 0 0.25rem;
|
|
||||||
font-size: inherit;
|
|
||||||
}
|
|
||||||
.fc-post-card__thumbs {
|
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-wrap: wrap;
|
flex-direction: column;
|
||||||
gap: 0.4rem;
|
gap: 16px;
|
||||||
align-items: center;
|
|
||||||
margin: 0.5rem 0;
|
|
||||||
}
|
}
|
||||||
.fc-post-card__thumb { line-height: 0; }
|
@container (min-width: 800px) {
|
||||||
.fc-post-card__thumb-img {
|
.fc-post-card__body {
|
||||||
border-radius: 4px;
|
flex-direction: row;
|
||||||
|
gap: 24px;
|
||||||
|
}
|
||||||
|
.fc-post-card__media {
|
||||||
|
flex: 0 0 50%;
|
||||||
|
}
|
||||||
|
.fc-post-card__text {
|
||||||
|
flex: 1 1 0;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-post-card__hero {
|
||||||
|
width: 100%;
|
||||||
|
aspect-ratio: 16 / 10;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
border-radius: 6px;
|
||||||
}
|
}
|
||||||
.fc-post-card__more-thumbs {
|
.fc-post-card__hero img {
|
||||||
display: inline-flex;
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
object-fit: cover;
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-post-card__rail {
|
||||||
|
display: flex;
|
||||||
|
gap: 6px;
|
||||||
|
margin-top: 6px;
|
||||||
|
}
|
||||||
|
.fc-post-card__rail-cell {
|
||||||
|
width: 80px;
|
||||||
|
height: 80px;
|
||||||
|
overflow: hidden;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
.fc-post-card__rail-cell img {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
object-fit: cover;
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
.fc-post-card__rail-more {
|
||||||
|
width: 80px;
|
||||||
|
height: 80px;
|
||||||
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
width: 96px;
|
|
||||||
height: 96px;
|
|
||||||
border: 1px dashed rgb(var(--v-theme-on-surface-variant));
|
border: 1px dashed rgb(var(--v-theme-on-surface-variant));
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
}
|
}
|
||||||
.fc-post-card__more-thumbs:hover {
|
|
||||||
color: rgb(var(--v-theme-accent));
|
.fc-post-card__title {
|
||||||
border-color: rgb(var(--v-theme-accent));
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 500;
|
||||||
|
margin: 0 0 8px 0;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
display: -webkit-box;
|
||||||
|
-webkit-line-clamp: 2;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
overflow: hidden;
|
||||||
}
|
}
|
||||||
.fc-post-card__attachments {
|
.fc-post-card__title--missing {
|
||||||
display: flex;
|
font-style: italic;
|
||||||
flex-wrap: wrap;
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
gap: 0.4rem;
|
|
||||||
margin-top: 0.6rem;
|
|
||||||
}
|
}
|
||||||
.fc-post-card__att {
|
@container (min-width: 800px) {
|
||||||
|
.fc-post-card__title {
|
||||||
|
font-size: 20px;
|
||||||
|
-webkit-line-clamp: 1;
|
||||||
|
white-space: nowrap;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-post-card__desc {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
line-height: 1.5;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
margin: 0 0 12px 0;
|
||||||
|
display: -webkit-box;
|
||||||
|
-webkit-line-clamp: 3;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.fc-post-card__desc--missing {
|
||||||
|
font-style: italic;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
@container (min-width: 800px) {
|
||||||
|
.fc-post-card__desc {
|
||||||
|
-webkit-line-clamp: 5;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-post-card__atts {
|
||||||
display: inline-flex;
|
display: inline-flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 0.35rem;
|
gap: 6px;
|
||||||
padding: 0.2rem 0.5rem;
|
font-size: 0.85rem;
|
||||||
border: 1px solid rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
border-radius: 999px;
|
|
||||||
color: rgb(var(--v-theme-on-surface));
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.82rem;
|
|
||||||
}
|
}
|
||||||
.fc-post-card__att:hover {
|
.fc-post-card__att-icon { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
color: rgb(var(--v-theme-accent));
|
|
||||||
border-color: rgb(var(--v-theme-accent));
|
|
||||||
}
|
|
||||||
.fc-post-card__att-size { color: rgb(var(--v-theme-on-surface-variant)); }
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-post-empty">
|
||||||
|
<v-icon size="48" class="fc-post-empty__icon">mdi-image-off-outline</v-icon>
|
||||||
|
<div class="fc-post-empty__text">No images attached to this post</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
// No props — pure presentational placeholder.
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-post-empty {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 32px 16px;
|
||||||
|
border: 1px dashed rgb(var(--v-theme-on-surface-variant));
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(20, 23, 26, 0.3);
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
min-height: 200px;
|
||||||
|
}
|
||||||
|
.fc-post-empty__icon {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
opacity: 0.6;
|
||||||
|
}
|
||||||
|
.fc-post-empty__text {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-post-grid">
|
||||||
|
<button
|
||||||
|
v-for="(t, idx) in thumbnails"
|
||||||
|
:key="t.image_id"
|
||||||
|
type="button"
|
||||||
|
class="fc-post-grid__cell"
|
||||||
|
:aria-label="`Open image ${idx + 1} of ${thumbnails.length}`"
|
||||||
|
@click="openImage(t.image_id, idx)"
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
:src="t.thumbnail_url"
|
||||||
|
:alt="`thumbnail ${idx + 1}`"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
thumbnails: { type: Array, required: true }, // [{ image_id, thumbnail_url, ... }]
|
||||||
|
})
|
||||||
|
|
||||||
|
const modal = useModalStore()
|
||||||
|
|
||||||
|
const imageIds = computed(() => props.thumbnails.map(t => t.image_id))
|
||||||
|
|
||||||
|
function openImage (id, idx) {
|
||||||
|
modal.open(id, { postImageIds: imageIds.value, initialIndex: idx })
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-post-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
.fc-post-grid__cell {
|
||||||
|
aspect-ratio: 4 / 3;
|
||||||
|
overflow: hidden;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
background: rgb(var(--v-theme-background));
|
||||||
|
transition: transform 0.15s ease, box-shadow 0.15s ease;
|
||||||
|
}
|
||||||
|
.fc-post-grid__cell:hover {
|
||||||
|
transform: scale(1.02);
|
||||||
|
box-shadow: 0 0 0 2px rgb(var(--v-theme-accent));
|
||||||
|
}
|
||||||
|
.fc-post-grid__cell img {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
object-fit: cover;
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
<template>
|
||||||
|
<v-dialog
|
||||||
|
:model-value="store.isOpen"
|
||||||
|
max-width="1100"
|
||||||
|
scrollable
|
||||||
|
@update:model-value="(v) => { if (!v) close() }"
|
||||||
|
>
|
||||||
|
<v-card v-if="store.currentPost" class="fc-post-modal">
|
||||||
|
<div class="fc-post-modal__head">
|
||||||
|
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
||||||
|
<RouterLink
|
||||||
|
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||||
|
class="fc-post-modal__artist"
|
||||||
|
@click="close"
|
||||||
|
>{{ post.artist.name }}</RouterLink>
|
||||||
|
<span class="fc-post-modal__date" :title="absoluteDate">
|
||||||
|
{{ relativeDate }}
|
||||||
|
</span>
|
||||||
|
<span v-if="post.thumbnails?.length" class="fc-post-modal__meta">
|
||||||
|
· {{ post.thumbnails.length }} image{{ post.thumbnails.length === 1 ? '' : 's' }}
|
||||||
|
</span>
|
||||||
|
<span v-if="post.attachments?.length" class="fc-post-modal__meta">
|
||||||
|
· {{ post.attachments.length }} attachment{{ post.attachments.length === 1 ? '' : 's' }}
|
||||||
|
</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
v-if="post.post_url"
|
||||||
|
:href="post.post_url" target="_blank" rel="noopener"
|
||||||
|
icon="mdi-open-in-new" size="small" variant="text"
|
||||||
|
:aria-label="`open original post on ${post.source.platform}`"
|
||||||
|
/>
|
||||||
|
<v-btn
|
||||||
|
icon="mdi-close" size="small" variant="text"
|
||||||
|
aria-label="Close"
|
||||||
|
@click="close"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-card-text class="fc-post-modal__body">
|
||||||
|
<section v-if="post.thumbnails?.length" class="fc-post-modal__sec">
|
||||||
|
<PostImageGrid :thumbnails="post.thumbnails" />
|
||||||
|
<div v-if="!store.detailLoaded" class="fc-post-modal__loading-hint">
|
||||||
|
Loading full image list…
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="post.post_title" class="fc-post-modal__sec">
|
||||||
|
<h2 class="fc-post-modal__title">{{ post.post_title }}</h2>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="descriptionHtml" class="fc-post-modal__sec">
|
||||||
|
<div class="fc-post-modal__desc" v-html="descriptionHtml" />
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="post.attachments?.length" class="fc-post-modal__sec">
|
||||||
|
<h3 class="fc-post-modal__h3">Attachments</h3>
|
||||||
|
<div class="fc-post-modal__atts">
|
||||||
|
<a
|
||||||
|
v-for="att in post.attachments" :key="att.id"
|
||||||
|
:href="att.download_url" download
|
||||||
|
class="fc-post-modal__att"
|
||||||
|
>
|
||||||
|
<v-icon size="small" class="fc-post-modal__att-icon">mdi-paperclip</v-icon>
|
||||||
|
<span class="fc-post-modal__att-name">{{ att.original_filename }}</span>
|
||||||
|
<span class="fc-post-modal__att-size">({{ formatBytes(att.size_bytes) }})</span>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import { usePostModalStore } from '../../stores/postModal.js'
|
||||||
|
import { sanitizeHtml } from '../../utils/htmlSanitize.js'
|
||||||
|
import PostImageGrid from './PostImageGrid.vue'
|
||||||
|
|
||||||
|
const store = usePostModalStore()
|
||||||
|
const post = computed(() => store.currentPost || {})
|
||||||
|
|
||||||
|
const sortDateIso = computed(() => post.value.post_date || post.value.downloaded_at)
|
||||||
|
const absoluteDate = computed(() => sortDateIso.value
|
||||||
|
? new Date(sortDateIso.value).toLocaleString()
|
||||||
|
: '')
|
||||||
|
const relativeDate = computed(() => {
|
||||||
|
if (!sortDateIso.value) return ''
|
||||||
|
const then = new Date(sortDateIso.value).getTime()
|
||||||
|
const diff = (Date.now() - then) / 1000
|
||||||
|
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||||
|
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||||
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||||
|
if (diff < 86400 * 30) return `${Math.floor(diff / 86400)}d ago`
|
||||||
|
return new Date(sortDateIso.value).toLocaleDateString()
|
||||||
|
})
|
||||||
|
|
||||||
|
const descriptionHtml = computed(() => {
|
||||||
|
// Detail endpoint returns description_full as plain text (the
|
||||||
|
// existing service uses html_to_plain on the stored description).
|
||||||
|
// Render the plain text in <p> wrappers; the sanitizer below is
|
||||||
|
// defensive for the case where the backend ever switches to raw HTML.
|
||||||
|
const raw = post.value.description_full || post.value.description_plain
|
||||||
|
if (!raw) return ''
|
||||||
|
if (/[<>]/.test(raw)) return sanitizeHtml(raw)
|
||||||
|
const esc = raw
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>')
|
||||||
|
return esc
|
||||||
|
.split(/\n\s*\n/)
|
||||||
|
.map(p => `<p>${p.replace(/\n/g, '<br>')}</p>`)
|
||||||
|
.join('')
|
||||||
|
})
|
||||||
|
|
||||||
|
function close () {
|
||||||
|
store.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes (n) {
|
||||||
|
if (!n) return '0 B'
|
||||||
|
if (n < 1024) return `${n} B`
|
||||||
|
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
|
||||||
|
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`
|
||||||
|
return `${(n / 1024 / 1024 / 1024).toFixed(1)} GB`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-post-modal {
|
||||||
|
background: rgb(var(--v-theme-surface));
|
||||||
|
max-height: 90vh;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
.fc-post-modal__head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
padding: 12px 16px;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||||
|
}
|
||||||
|
.fc-post-modal__artist {
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
text-decoration: none;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.fc-post-modal__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
|
.fc-post-modal__date { white-space: nowrap; }
|
||||||
|
.fc-post-modal__meta { white-space: nowrap; }
|
||||||
|
.fc-post-modal__body {
|
||||||
|
padding: 16px;
|
||||||
|
overflow-y: auto;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
}
|
||||||
|
.fc-post-modal__sec {
|
||||||
|
margin-bottom: 24px;
|
||||||
|
}
|
||||||
|
.fc-post-modal__sec:last-child { margin-bottom: 0; }
|
||||||
|
.fc-post-modal__title {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 24px;
|
||||||
|
font-weight: 500;
|
||||||
|
margin: 0;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
|
.fc-post-modal__h3 {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 16px;
|
||||||
|
font-weight: 500;
|
||||||
|
margin: 0 0 8px 0;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
|
.fc-post-modal__desc {
|
||||||
|
font-size: 0.95rem;
|
||||||
|
line-height: 1.55;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
|
.fc-post-modal__desc :deep(p) { margin: 0 0 12px 0; }
|
||||||
|
.fc-post-modal__desc :deep(a) { color: rgb(var(--v-theme-accent)); }
|
||||||
|
.fc-post-modal__loading-hint {
|
||||||
|
margin-top: 8px;
|
||||||
|
font-size: 0.8rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-post-modal__atts {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.fc-post-modal__att {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
padding: 6px 12px;
|
||||||
|
border: 1px solid rgb(var(--v-theme-on-surface-variant));
|
||||||
|
border-radius: 999px;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
text-decoration: none;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
.fc-post-modal__att:hover {
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
border-color: rgb(var(--v-theme-accent));
|
||||||
|
}
|
||||||
|
.fc-post-modal__att-size { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-backup-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-database-export" size="small" />
|
||||||
|
<span>Backups</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
|
pg_dump for the database (fast, nightly-schedulable);
|
||||||
|
tar+zstd for the images (slow, manual only). Files live in
|
||||||
|
<code>/images/_backups/</code>. Tag a backup to protect it
|
||||||
|
from autoprune.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="store.lastError"
|
||||||
|
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>
|
||||||
|
{{ store.lastError }}
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<!-- Database section -->
|
||||||
|
<h3 class="fc-section-title">Database</h3>
|
||||||
|
<div class="fc-settings-row mb-3">
|
||||||
|
<v-switch
|
||||||
|
:model-value="settings.backup_db_nightly_enabled"
|
||||||
|
color="accent" density="compact" hide-details
|
||||||
|
label="Nightly"
|
||||||
|
@update:model-value="onSettingChange('backup_db_nightly_enabled', $event)"
|
||||||
|
/>
|
||||||
|
<v-text-field
|
||||||
|
:model-value="settings.backup_db_nightly_hour_utc"
|
||||||
|
label="Hour (UTC)" type="number" min="0" max="23"
|
||||||
|
density="compact" hide-details style="max-width: 110px;"
|
||||||
|
@update:model-value="onSettingChange('backup_db_nightly_hour_utc', Number($event))"
|
||||||
|
/>
|
||||||
|
<v-text-field
|
||||||
|
:model-value="settings.backup_db_keep_last_n"
|
||||||
|
label="Keep last" type="number" min="1" max="365"
|
||||||
|
density="compact" hide-details style="max-width: 110px;"
|
||||||
|
@update:model-value="onSettingChange('backup_db_keep_last_n', Number($event))"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-database-arrow-up" class="mb-3"
|
||||||
|
:loading="dbTriggering"
|
||||||
|
@click="onTrigger('db')"
|
||||||
|
>Run DB backup now</v-btn>
|
||||||
|
|
||||||
|
<BackupRunsTable
|
||||||
|
:runs="store.dbRuns"
|
||||||
|
@restore="onRestore"
|
||||||
|
@delete="onDelete"
|
||||||
|
@tag="onTag"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<v-divider class="my-4" />
|
||||||
|
|
||||||
|
<!-- Images section -->
|
||||||
|
<h3 class="fc-section-title">Images</h3>
|
||||||
|
<div class="fc-settings-row mb-3">
|
||||||
|
<v-text-field
|
||||||
|
:model-value="settings.backup_images_keep_last_n"
|
||||||
|
label="Keep last" type="number" min="1" max="100"
|
||||||
|
density="compact" hide-details style="max-width: 110px;"
|
||||||
|
@update:model-value="onSettingChange('backup_images_keep_last_n', Number($event))"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-folder-zip" class="mb-3"
|
||||||
|
:loading="imagesTriggering"
|
||||||
|
@click="onTrigger('images')"
|
||||||
|
>Run images backup now</v-btn>
|
||||||
|
<span class="text-caption fc-muted ml-2">
|
||||||
|
ⓘ Hours for a 200GB+ library.
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<BackupRunsTable
|
||||||
|
:runs="store.imagesRuns"
|
||||||
|
@restore="onRestore"
|
||||||
|
@delete="onDelete"
|
||||||
|
@tag="onTag"
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<BackupConfirmModal
|
||||||
|
v-model="confirmOpen"
|
||||||
|
:action="confirmAction"
|
||||||
|
:kind="confirmKind"
|
||||||
|
:run-id="confirmRunId"
|
||||||
|
:description="confirmDescription"
|
||||||
|
@confirm="onConfirmSubmit"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useBackupStore } from '../../stores/backup.js'
|
||||||
|
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import BackupRunsTable from './BackupRunsTable.vue'
|
||||||
|
|
||||||
|
const store = useBackupStore()
|
||||||
|
|
||||||
|
const settings = computed(() => store.settings || {
|
||||||
|
backup_db_nightly_enabled: false,
|
||||||
|
backup_db_nightly_hour_utc: 3,
|
||||||
|
backup_db_keep_last_n: 14,
|
||||||
|
backup_images_keep_last_n: 3,
|
||||||
|
})
|
||||||
|
|
||||||
|
const dbTriggering = ref(false)
|
||||||
|
const imagesTriggering = ref(false)
|
||||||
|
const confirmOpen = ref(false)
|
||||||
|
const confirmAction = ref('restore')
|
||||||
|
const confirmKind = ref('db')
|
||||||
|
const confirmRunId = ref(0)
|
||||||
|
const confirmDescription = ref('')
|
||||||
|
|
||||||
|
let pollId = null
|
||||||
|
function pollOnce() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadRuns('db')
|
||||||
|
store.loadRuns('images')
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
store.loadSettings()
|
||||||
|
pollOnce()
|
||||||
|
pollId = setInterval(pollOnce, 5000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (pollId) { clearInterval(pollId); pollId = null }
|
||||||
|
})
|
||||||
|
|
||||||
|
async function onTrigger(kind) {
|
||||||
|
const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
|
||||||
|
triggeringRef.value = true
|
||||||
|
try {
|
||||||
|
await store.triggerBackup(kind)
|
||||||
|
pollOnce()
|
||||||
|
} finally {
|
||||||
|
triggeringRef.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onSettingChange(field, value) {
|
||||||
|
await store.patchSettings({ [field]: value })
|
||||||
|
}
|
||||||
|
|
||||||
|
function _openConfirm(action, run) {
|
||||||
|
confirmAction.value = action
|
||||||
|
confirmKind.value = run.kind
|
||||||
|
confirmRunId.value = run.id
|
||||||
|
confirmDescription.value = (
|
||||||
|
action === 'restore'
|
||||||
|
? `Source artifact: ${run.kind === 'db' ? run.sql_path : run.tar_path}`
|
||||||
|
: 'Removes the BackupRun row and unlinks the artifact files.'
|
||||||
|
)
|
||||||
|
confirmOpen.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
function onRestore(run) { _openConfirm('restore', run) }
|
||||||
|
function onDelete(run) { _openConfirm('delete', run) }
|
||||||
|
|
||||||
|
async function onTag(run) {
|
||||||
|
const next = prompt(
|
||||||
|
`Tag for backup #${run.id} (blank to clear):`,
|
||||||
|
run.tag || '',
|
||||||
|
)
|
||||||
|
if (next === null) return
|
||||||
|
await store.setTag(run.id, next.trim() || null)
|
||||||
|
pollOnce()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmSubmit(token) {
|
||||||
|
try {
|
||||||
|
if (confirmAction.value === 'restore') {
|
||||||
|
await store.restore(confirmRunId.value, token)
|
||||||
|
} else {
|
||||||
|
await store.deleteRun(confirmRunId.value, token)
|
||||||
|
}
|
||||||
|
pollOnce()
|
||||||
|
} catch {
|
||||||
|
// store.lastError already updated; surfaced in v-alert above.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-backup-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-section-title {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-size: 16px; font-weight: 500;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.fc-settings-row {
|
||||||
|
display: flex; gap: 12px; align-items: center; flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
<template>
|
||||||
|
<v-table density="compact" class="fc-backup-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>When</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th class="text-right">Duration</th>
|
||||||
|
<th class="text-right">Size</th>
|
||||||
|
<th>Tag</th>
|
||||||
|
<th class="text-right">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in runs" :key="r.id">
|
||||||
|
<td class="fc-tabular" :title="r.started_at">
|
||||||
|
{{ formatRelative(r.started_at) }}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<v-icon size="small" :color="statusColor(r.status)">
|
||||||
|
{{ statusIcon(r.status) }}
|
||||||
|
</v-icon>
|
||||||
|
{{ r.status }}
|
||||||
|
</td>
|
||||||
|
<td class="text-right fc-tabular">{{ formatDuration(r) }}</td>
|
||||||
|
<td class="text-right fc-tabular">{{ formatBytes(r.size_bytes) }}</td>
|
||||||
|
<td>
|
||||||
|
<v-chip
|
||||||
|
v-if="r.tag" size="x-small" color="accent" variant="tonal"
|
||||||
|
>{{ r.tag }}</v-chip>
|
||||||
|
<span v-else class="fc-muted">—</span>
|
||||||
|
</td>
|
||||||
|
<td class="text-right">
|
||||||
|
<v-menu>
|
||||||
|
<template #activator="{ props: act }">
|
||||||
|
<v-btn icon="mdi-dots-vertical" size="x-small" variant="text" v-bind="act" />
|
||||||
|
</template>
|
||||||
|
<v-list density="compact">
|
||||||
|
<v-list-item
|
||||||
|
:title="r.tag ? 'Untag' : 'Tag…'"
|
||||||
|
@click="$emit('tag', r)"
|
||||||
|
/>
|
||||||
|
<v-list-item
|
||||||
|
title="Restore…" :disabled="r.status !== 'ok'"
|
||||||
|
@click="$emit('restore', r)"
|
||||||
|
/>
|
||||||
|
<v-list-item
|
||||||
|
title="Delete…"
|
||||||
|
@click="$emit('delete', r)"
|
||||||
|
/>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!runs.length">
|
||||||
|
<td colspan="6" class="text-center fc-muted py-4">No backups yet.</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
defineProps({ runs: { type: Array, default: () => [] } })
|
||||||
|
defineEmits(['restore', 'delete', 'tag'])
|
||||||
|
|
||||||
|
function statusIcon(s) {
|
||||||
|
return {
|
||||||
|
ok: 'mdi-check-circle', error: 'mdi-close-circle',
|
||||||
|
running: 'mdi-timer-sand', restoring: 'mdi-restore-clock',
|
||||||
|
restored: 'mdi-restore', pending: 'mdi-clock-outline',
|
||||||
|
}[s] || 'mdi-help-circle'
|
||||||
|
}
|
||||||
|
function statusColor(s) {
|
||||||
|
return {
|
||||||
|
ok: 'success', error: 'error',
|
||||||
|
running: 'accent', restoring: 'info',
|
||||||
|
restored: 'info', pending: 'on-surface-variant',
|
||||||
|
}[s] || 'on-surface-variant'
|
||||||
|
}
|
||||||
|
function formatDuration(r) {
|
||||||
|
if (r.duration_seconds == null) return '—'
|
||||||
|
const s = r.duration_seconds
|
||||||
|
if (s < 60) return `${s}s`
|
||||||
|
if (s < 3600) return `${Math.floor(s / 60)}m ${s % 60}s`
|
||||||
|
const h = Math.floor(s / 3600); const m = Math.floor((s % 3600) / 60)
|
||||||
|
return `${h}h ${m}m`
|
||||||
|
}
|
||||||
|
function formatBytes(b) {
|
||||||
|
if (b == null) return '—'
|
||||||
|
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
|
||||||
|
let i = 0; let v = b
|
||||||
|
while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ }
|
||||||
|
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
||||||
|
}
|
||||||
|
function formatRelative(iso) {
|
||||||
|
if (!iso) return '—'
|
||||||
|
const then = new Date(iso).getTime()
|
||||||
|
const diff = Math.max(0, (Date.now() - then) / 1000)
|
||||||
|
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||||
|
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||||
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||||
|
return `${Math.floor(diff / 86400)}d ago`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-backup-table { background: transparent; }
|
||||||
|
.fc-tabular { font-variant-numeric: tabular-nums; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -34,11 +34,18 @@
|
|||||||
|
|
||||||
<template v-else-if="manifest?.installed">
|
<template v-else-if="manifest?.installed">
|
||||||
<div class="fc-ext-install mt-3">
|
<div class="fc-ext-install mt-3">
|
||||||
|
<!-- Install button: direct :href anchor click (no programmatic
|
||||||
|
window.location.assign). Firefox's XPI-install gesture
|
||||||
|
requires a user-clicked anchor pointing at an
|
||||||
|
application/x-xpinstall response; programmatic navigation
|
||||||
|
sometimes triggered nothing instead of the install dialog
|
||||||
|
(operator-flagged 2026-05-26). No `download` attribute —
|
||||||
|
that would force a save dialog instead of install. -->
|
||||||
<v-btn
|
<v-btn
|
||||||
v-if="isFirefox"
|
v-if="isFirefox"
|
||||||
color="accent" variant="flat" rounded="pill"
|
color="accent" variant="flat" rounded="pill"
|
||||||
prepend-icon="mdi-firefox"
|
prepend-icon="mdi-firefox"
|
||||||
@click="installXpi"
|
:href="manifest.latest_url"
|
||||||
>Install Firefox extension</v-btn>
|
>Install Firefox extension</v-btn>
|
||||||
|
|
||||||
<v-btn
|
<v-btn
|
||||||
@@ -103,6 +110,7 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, ref } from 'vue'
|
import { computed, onMounted, ref } from 'vue'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
import { useApi } from '../../composables/useApi.js'
|
||||||
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
|
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
|
|
||||||
@@ -146,11 +154,6 @@ async function loadKey() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function installXpi() {
|
|
||||||
if (!manifest.value?.latest_url) return
|
|
||||||
window.location.assign(manifest.value.latest_url)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function rotateKey() {
|
async function rotateKey() {
|
||||||
rotating.value = true
|
rotating.value = true
|
||||||
try {
|
try {
|
||||||
@@ -170,7 +173,7 @@ async function rotateKey() {
|
|||||||
|
|
||||||
async function copy(text, label) {
|
async function copy(text, label) {
|
||||||
try {
|
try {
|
||||||
await navigator.clipboard.writeText(text)
|
await copyText(text)
|
||||||
window.__fcToast?.({ text: `${label} copied.`, type: 'success' })
|
window.__fcToast?.({ text: `${label} copied.`, type: 'success' })
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
|
|||||||
@@ -17,6 +17,12 @@
|
|||||||
>
|
>
|
||||||
Retry failed
|
Retry failed
|
||||||
</v-btn>
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
variant="text" rounded="pill" size="small" color="warning"
|
||||||
|
:disabled="!hasStuck" @click="onClearStuckOpen"
|
||||||
|
>
|
||||||
|
Clear stuck…
|
||||||
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
variant="text" rounded="pill" size="small" color="error"
|
variant="text" rounded="pill" size="small" color="error"
|
||||||
@click="onClearOpen"
|
@click="onClearOpen"
|
||||||
@@ -39,9 +45,11 @@
|
|||||||
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
||||||
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
||||||
<template #item.error="{ item }">
|
<template #item.error="{ item }">
|
||||||
<span v-if="item.error" :title="item.error" class="text-caption">
|
<button
|
||||||
{{ shorten(item.error, 60) }}
|
v-if="item.error" type="button" class="fc-err-link text-caption"
|
||||||
</span>
|
@click="openError(`Task ${item.id} failed`, item.error)"
|
||||||
|
title="Click for full error"
|
||||||
|
>{{ shorten(item.error, 60) }}</button>
|
||||||
</template>
|
</template>
|
||||||
</v-data-table-virtual>
|
</v-data-table-virtual>
|
||||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||||
@@ -69,17 +77,62 @@
|
|||||||
</v-card-actions>
|
</v-card-actions>
|
||||||
</v-card>
|
</v-card>
|
||||||
</v-dialog>
|
</v-dialog>
|
||||||
|
|
||||||
|
<v-dialog v-model="clearStuckDialog" max-width="480">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Clear stuck tasks</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||||
|
Force every <strong>pending / queued / processing</strong> task to
|
||||||
|
<strong>failed</strong> and finalize any active batch that
|
||||||
|
has no remaining work. Use this when the automatic recovery
|
||||||
|
sweep keeps re-queueing the same row (e.g., corrupt file in
|
||||||
|
an autoretry loop, or worker model missing).
|
||||||
|
</v-alert>
|
||||||
|
<p class="text-body-2">
|
||||||
|
Tasks remain in the database with status=<code>failed</code>;
|
||||||
|
click <em>Retry failed</em> once the underlying cause is
|
||||||
|
resolved to re-queue them.
|
||||||
|
</p>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn @click="clearStuckDialog = false">Cancel</v-btn>
|
||||||
|
<v-btn color="warning" rounded="pill" @click="onClearStuckConfirm">Clear stuck</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
|
||||||
|
<ErrorDetailModal
|
||||||
|
v-model="showErrorModal"
|
||||||
|
:title="errorModalTitle"
|
||||||
|
:message="errorModalMessage"
|
||||||
|
/>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import { useImportStore } from '../../stores/import.js'
|
import { useImportStore } from '../../stores/import.js'
|
||||||
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
|
|
||||||
const store = useImportStore()
|
const store = useImportStore()
|
||||||
const statusFilter = ref(null)
|
const statusFilter = ref(null)
|
||||||
const clearDialog = ref(false)
|
const clearDialog = ref(false)
|
||||||
|
// Click-to-open modal for full error text (operator-flagged 2026-05-26
|
||||||
|
// — the prior :title="..." tooltip cramped multi-line SQLAlchemy
|
||||||
|
// tracebacks into an unusable popup with no copy-paste affordance).
|
||||||
|
const showErrorModal = ref(false)
|
||||||
|
const errorModalTitle = ref('')
|
||||||
|
const errorModalMessage = ref('')
|
||||||
|
|
||||||
|
function openError(title, message) {
|
||||||
|
errorModalTitle.value = title
|
||||||
|
errorModalMessage.value = message || ''
|
||||||
|
showErrorModal.value = true
|
||||||
|
}
|
||||||
const clearAgeDays = ref(7)
|
const clearAgeDays = ref(7)
|
||||||
|
const clearStuckDialog = ref(false)
|
||||||
|
|
||||||
const statusOptions = [
|
const statusOptions = [
|
||||||
{ title: 'All', value: null },
|
{ title: 'All', value: null },
|
||||||
@@ -100,6 +153,9 @@ const headers = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||||
|
const hasStuck = computed(() => store.tasks.some(
|
||||||
|
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||||
|
))
|
||||||
|
|
||||||
function statusColor(s) {
|
function statusColor(s) {
|
||||||
return {
|
return {
|
||||||
@@ -138,4 +194,27 @@ async function onClearConfirm() {
|
|||||||
await store.clearCompleted(clearAgeDays.value)
|
await store.clearCompleted(clearAgeDays.value)
|
||||||
clearDialog.value = false
|
clearDialog.value = false
|
||||||
}
|
}
|
||||||
|
function onClearStuckOpen() { clearStuckDialog.value = true }
|
||||||
|
async function onClearStuckConfirm() {
|
||||||
|
await store.clearStuck()
|
||||||
|
clearStuckDialog.value = false
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-err-link {
|
||||||
|
/* Truncated error preview as a clickable button — opens
|
||||||
|
ErrorDetailModal with the full text. Inherits the row's font
|
||||||
|
sizing so it doesn't visually drift from the prior tooltip-bearing
|
||||||
|
span. */
|
||||||
|
color: rgb(var(--v-theme-error, 220 80 80));
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
font: inherit;
|
||||||
|
text-align: left;
|
||||||
|
text-decoration: underline dotted;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-err-link:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
|
|||||||
@@ -2,31 +2,71 @@
|
|||||||
<v-card>
|
<v-card>
|
||||||
<v-card-title>Trigger scan</v-card-title>
|
<v-card-title>Trigger scan</v-card-title>
|
||||||
<v-card-text>
|
<v-card-text>
|
||||||
<div v-if="store.activeBatch" class="d-flex align-center" style="gap: 12px;">
|
<div v-if="store.activeBatch" class="d-flex align-center mb-3" style="gap: 12px;">
|
||||||
<v-progress-circular
|
<v-progress-circular
|
||||||
indeterminate color="accent" size="20"
|
indeterminate color="accent" size="20"
|
||||||
/>
|
/>
|
||||||
<span>
|
<span>
|
||||||
Scanning {{ store.activeBatch.source_path }} —
|
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
|
||||||
|
{{ store.activeBatch.source_path || '/import' }} —
|
||||||
imported {{ store.activeBatch.imported }},
|
imported {{ store.activeBatch.imported }},
|
||||||
|
<template v-if="store.activeBatch.scan_mode === 'deep'">
|
||||||
|
refreshed {{ store.activeBatch.refreshed || 0 }},
|
||||||
|
</template>
|
||||||
skipped {{ store.activeBatch.skipped }},
|
skipped {{ store.activeBatch.skipped }},
|
||||||
failed {{ store.activeBatch.failed }} /
|
failed {{ store.activeBatch.failed }} /
|
||||||
{{ store.activeBatch.total_files }} files
|
{{ store.activeBatch.total_files }} files
|
||||||
</span>
|
</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
variant="text" rounded="pill" size="small" color="warning"
|
||||||
|
:loading="clearing" @click="onClearStuck"
|
||||||
|
>
|
||||||
|
Clear stuck
|
||||||
|
</v-btn>
|
||||||
</div>
|
</div>
|
||||||
<div v-else>
|
|
||||||
<p class="text-body-2 mb-3">
|
<p class="text-body-2 mb-3">
|
||||||
Run a quick scan of the import directory. Deep scan (pHash dedup,
|
<span v-if="!store.activeBatch">
|
||||||
archives) lands in FC-2d.
|
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
|
||||||
</p>
|
new files only.
|
||||||
<v-btn color="primary" rounded="pill" @click="trigger" :loading="busy">
|
<strong>Deep scan</strong> additionally re-walks already-imported
|
||||||
|
files so updated sidecar metadata (post title/date/attribution) and
|
||||||
|
previously-NULL phashes / artist links get refreshed. Use after
|
||||||
|
bulk-downloading fresh sidecars for existing content. Both modes
|
||||||
|
route non-media + sidecar pairs through PostAttachment capture.
|
||||||
|
</span>
|
||||||
|
<span v-else>
|
||||||
|
An active batch is in progress. Wait for it to finish, or click
|
||||||
|
<em>Clear stuck</em> above if it has been wedged with no
|
||||||
|
measurable progress.
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="d-flex flex-wrap" style="gap: 12px;">
|
||||||
|
<v-btn
|
||||||
|
color="primary" rounded="pill"
|
||||||
|
:disabled="!!store.activeBatch"
|
||||||
|
:loading="busy === 'quick'"
|
||||||
|
@click="trigger('quick')"
|
||||||
|
>
|
||||||
<v-icon start>mdi-magnify-scan</v-icon>
|
<v-icon start>mdi-magnify-scan</v-icon>
|
||||||
Quick scan
|
Quick scan
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
|
<v-btn
|
||||||
{{ store.triggerError }}
|
color="secondary" rounded="pill" variant="tonal"
|
||||||
</v-alert>
|
:disabled="!!store.activeBatch"
|
||||||
|
:loading="busy === 'deep'"
|
||||||
|
@click="trigger('deep')"
|
||||||
|
>
|
||||||
|
<v-icon start>mdi-magnify-plus-outline</v-icon>
|
||||||
|
Deep scan
|
||||||
|
</v-btn>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
|
||||||
|
{{ store.triggerError }}
|
||||||
|
</v-alert>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
@@ -36,10 +76,22 @@ import { ref } from 'vue'
|
|||||||
import { useImportStore } from '../../stores/import.js'
|
import { useImportStore } from '../../stores/import.js'
|
||||||
|
|
||||||
const store = useImportStore()
|
const store = useImportStore()
|
||||||
const busy = ref(false)
|
const busy = ref(null)
|
||||||
|
const clearing = ref(false)
|
||||||
|
|
||||||
async function trigger() {
|
async function trigger(mode) {
|
||||||
busy.value = true
|
busy.value = mode
|
||||||
try { await store.triggerScan() } catch {} finally { busy.value = false }
|
try { await store.triggerScan(mode) } catch {} finally { busy.value = null }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onClearStuck() {
|
||||||
|
clearing.value = true
|
||||||
|
try {
|
||||||
|
await store.clearStuck()
|
||||||
|
} catch {
|
||||||
|
// store surfaces error via triggerError if needed
|
||||||
|
} finally {
|
||||||
|
clearing.value = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -1,18 +1,23 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-maint">
|
<div class="fc-maint">
|
||||||
<p class="text-body-2 mb-4">
|
<p class="text-body-2 mb-4">
|
||||||
Machine-assisted tagging controls. Backfill and centroid recompute run
|
Operational backfills and tagging controls. The ML backfill and centroid
|
||||||
nightly automatically; the allowlist auto-applies accepted tags to new
|
recompute run nightly automatically; the allowlist auto-applies accepted
|
||||||
and existing images.
|
tags to new and existing images. Use the cards below to trigger a
|
||||||
|
one-off pass.
|
||||||
</p>
|
</p>
|
||||||
<div class="fc-maint__grid">
|
<div class="fc-maint__grid">
|
||||||
<MLBackfillCard />
|
<MLBackfillCard />
|
||||||
<CentroidRecomputeCard />
|
<CentroidRecomputeCard />
|
||||||
|
<ThumbnailBackfillCard />
|
||||||
</div>
|
</div>
|
||||||
<MLThresholdSliders class="mt-4" />
|
<MLThresholdSliders class="mt-4" />
|
||||||
<AllowlistTable class="mt-4" />
|
<AllowlistTable class="mt-4" />
|
||||||
<AliasTable class="mt-4" />
|
<AliasTable class="mt-4" />
|
||||||
<BrowserExtensionCard class="mt-6" />
|
<BackupCard class="mt-6" />
|
||||||
|
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||||
|
operates on the existing library which fits the Cleanup-tab
|
||||||
|
theme, and clusters with the other audit cards. -->
|
||||||
<LegacyMigrationCard class="mt-6" />
|
<LegacyMigrationCard class="mt-6" />
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -20,10 +25,11 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import MLBackfillCard from './MLBackfillCard.vue'
|
import MLBackfillCard from './MLBackfillCard.vue'
|
||||||
import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
||||||
|
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||||
import MLThresholdSliders from './MLThresholdSliders.vue'
|
import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||||
import AllowlistTable from './AllowlistTable.vue'
|
import AllowlistTable from './AllowlistTable.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import BrowserExtensionCard from './BrowserExtensionCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
<template>
|
||||||
|
<v-table density="compact" class="fc-queues-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th class="text-right">Depth</th>
|
||||||
|
<th class="text-right">Workers</th>
|
||||||
|
<th v-if="!compact" class="text-right">Active</th>
|
||||||
|
<th class="text-right">Last min: ok / err</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="q in QUEUE_NAMES" :key="q">
|
||||||
|
<td>{{ q }}</td>
|
||||||
|
<td class="text-right fc-tabular" :class="depthClass(q)">
|
||||||
|
{{ formatDepth(q) }}
|
||||||
|
</td>
|
||||||
|
<td class="text-right fc-tabular">{{ workerCount(q) }}</td>
|
||||||
|
<td v-if="!compact" class="text-right fc-tabular">{{ activeCount(q) }}</td>
|
||||||
|
<td class="text-right fc-tabular">
|
||||||
|
<span class="fc-ok">{{ recentOk(q) }}</span>
|
||||||
|
<span class="fc-sep">/</span>
|
||||||
|
<span :class="recentErr(q) > 0 ? 'fc-err' : 'fc-muted'">
|
||||||
|
{{ recentErr(q) }}<span v-if="recentErr(q) > 0"> ⚠</span>
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
queues: { type: Object, default: null }, // store.queues
|
||||||
|
workers: { type: Object, default: null }, // store.workers
|
||||||
|
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
|
||||||
|
compact: { type: Boolean, default: false },
|
||||||
|
})
|
||||||
|
|
||||||
|
const QUEUE_NAMES = [
|
||||||
|
'default', 'import', 'thumbnail', 'ml',
|
||||||
|
'download', 'scan', 'maintenance',
|
||||||
|
]
|
||||||
|
|
||||||
|
function formatDepth(name) {
|
||||||
|
const d = props.queues?.queues?.[name]
|
||||||
|
if (d == null) return '—'
|
||||||
|
return d.toLocaleString()
|
||||||
|
}
|
||||||
|
|
||||||
|
function depthClass(name) {
|
||||||
|
const d = props.queues?.queues?.[name]
|
||||||
|
const workers = workerCount(name)
|
||||||
|
// Queue has depth but no workers → warning.
|
||||||
|
if (d != null && d > 0 && workers === 0) return 'fc-warn'
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function workerCount(name) {
|
||||||
|
if (!props.workers?.workers) return 0
|
||||||
|
let count = 0
|
||||||
|
for (const info of Object.values(props.workers.workers)) {
|
||||||
|
if (info.queues?.includes(name)) count++
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
function activeCount(name) {
|
||||||
|
if (!props.workers?.workers) return 0
|
||||||
|
// Active count is per-worker, not per-queue. Approximate: sum of
|
||||||
|
// active_count across workers that subscribe to this queue.
|
||||||
|
let count = 0
|
||||||
|
for (const info of Object.values(props.workers.workers)) {
|
||||||
|
if (info.queues?.includes(name)) count += info.active_count || 0
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
const recentByQueue = computed(() => {
|
||||||
|
const out = {}
|
||||||
|
for (const r of props.recentMinute) {
|
||||||
|
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
||||||
|
if (r.status === 'ok') out[r.queue].ok++
|
||||||
|
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
})
|
||||||
|
|
||||||
|
function recentOk(name) { return recentByQueue.value[name]?.ok || 0 }
|
||||||
|
function recentErr(name) { return recentByQueue.value[name]?.err || 0 }
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-queues-table { background: transparent; }
|
||||||
|
.fc-tabular {
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
font-feature-settings: 'tnum';
|
||||||
|
}
|
||||||
|
.fc-ok { color: rgb(var(--v-theme-success, 76 175 80)); }
|
||||||
|
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||||
|
.fc-warn { color: rgb(var(--v-theme-warning, 255 179 0)); font-weight: 500; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-sep {
|
||||||
|
margin: 0 4px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-activity-summary">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-pulse" size="small" />
|
||||||
|
<span>System activity (last min)</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" rounded="pill"
|
||||||
|
@click="$emit('open-activity')"
|
||||||
|
>
|
||||||
|
View activity
|
||||||
|
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
|
||||||
|
<v-card-text>
|
||||||
|
<v-alert
|
||||||
|
v-if="store.lastError"
|
||||||
|
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>
|
||||||
|
{{ store.lastError }}
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<QueuesTable
|
||||||
|
:queues="store.queues"
|
||||||
|
:workers="store.workers"
|
||||||
|
:recent-minute="store.recentMinute"
|
||||||
|
compact
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
|
defineEmits(['open-activity'])
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
let pollId = null
|
||||||
|
|
||||||
|
function pollOnce() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadQueues()
|
||||||
|
store.loadWorkers()
|
||||||
|
store.loadRecentMinute()
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
pollOnce()
|
||||||
|
pollId = setInterval(pollOnce, 5000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (pollId) { clearInterval(pollId); pollId = null }
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-activity-summary { border-radius: 8px; }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,315 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-activity">
|
||||||
|
<!-- Queues + workers pane -->
|
||||||
|
<v-card class="mb-4">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-pulse" size="small" />
|
||||||
|
<span>Queues + workers</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="text-caption fc-muted">
|
||||||
|
updated {{ formatRelative(store.queues?.fetched_at) }}
|
||||||
|
</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<QueuesTable
|
||||||
|
:queues="store.queues"
|
||||||
|
:workers="store.workers"
|
||||||
|
:recent-minute="store.recentMinute"
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<!-- Recent failures pane -->
|
||||||
|
<v-card class="mb-4">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-alert-circle-outline" size="small" />
|
||||||
|
<span>Recent failures (last 24h)</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="text-caption fc-muted">
|
||||||
|
{{ failureCount }} failures
|
||||||
|
</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<div v-if="errorTypes.length" class="mb-3 fc-pills">
|
||||||
|
<v-chip
|
||||||
|
v-for="t in errorTypes" :key="t.name"
|
||||||
|
:color="t.name === filterErrorType ? 'accent' : undefined"
|
||||||
|
size="small" variant="tonal" closable
|
||||||
|
@click="toggleErrorTypeFilter(t.name)"
|
||||||
|
@click:close="toggleErrorTypeFilter(t.name)"
|
||||||
|
>
|
||||||
|
{{ t.name }} × {{ t.count }}
|
||||||
|
</v-chip>
|
||||||
|
</div>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Time</th>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th>Task</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th>Error</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in filteredFailures" :key="r.id">
|
||||||
|
<td class="fc-tabular" :title="r.finished_at">
|
||||||
|
{{ formatRelative(r.finished_at) }}
|
||||||
|
</td>
|
||||||
|
<td>{{ r.queue }}</td>
|
||||||
|
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||||
|
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||||
|
<td>
|
||||||
|
<button
|
||||||
|
type="button" class="fc-err-link"
|
||||||
|
@click="openError(r)"
|
||||||
|
:title="'Click for full error'"
|
||||||
|
>{{ r.error_type }}</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!filteredFailures.length">
|
||||||
|
<td colspan="5" class="text-center fc-muted py-4">
|
||||||
|
No failures in the last 24h.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<!-- All activity pane -->
|
||||||
|
<v-card>
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-format-list-bulleted" size="small" />
|
||||||
|
<span>All recent activity</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-select
|
||||||
|
v-model="filterQueue"
|
||||||
|
:items="queueOptions" density="compact" hide-details
|
||||||
|
style="max-width: 180px;"
|
||||||
|
@update:model-value="onFilterChange"
|
||||||
|
/>
|
||||||
|
<v-select
|
||||||
|
v-model="filterStatus"
|
||||||
|
:items="statusOptions" density="compact" hide-details
|
||||||
|
style="max-width: 180px;"
|
||||||
|
@update:model-value="onFilterChange"
|
||||||
|
/>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" rounded="pill"
|
||||||
|
@click="onRefresh"
|
||||||
|
>
|
||||||
|
<v-icon start size="small">mdi-refresh</v-icon>
|
||||||
|
Refresh
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th></th>
|
||||||
|
<th>Time</th>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th>Task</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th class="text-right">Duration</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in store.runs" :key="r.id">
|
||||||
|
<td><v-icon size="small" :color="statusColor(r.status)">{{ statusIcon(r.status) }}</v-icon></td>
|
||||||
|
<td class="fc-tabular" :title="r.started_at">{{ formatRelative(r.started_at) }}</td>
|
||||||
|
<td>{{ r.queue }}</td>
|
||||||
|
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||||
|
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||||
|
<td class="text-right fc-tabular">{{ formatDuration(r.duration_ms) }}</td>
|
||||||
|
<td>{{ r.status }}</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!store.runs.length && !store.loading.runs">
|
||||||
|
<td colspan="7" class="text-center fc-muted py-4">
|
||||||
|
No activity yet.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
<div v-if="store.runsHasMore" class="d-flex justify-center py-3">
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small"
|
||||||
|
:loading="store.loading.runs"
|
||||||
|
@click="onLoadMore"
|
||||||
|
>Load more</v-btn>
|
||||||
|
</div>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<ErrorDetailModal
|
||||||
|
v-model="showErrorModal"
|
||||||
|
:row="errorModalRow"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
|
// Click-to-open modal for full error text. Replaces the unusable
|
||||||
|
// :title="..." tooltip (operator-flagged 2026-05-26: SQLAlchemy
|
||||||
|
// rollback + traceback content rendered as a cramped browser tooltip
|
||||||
|
// you couldn't copy from or scroll within).
|
||||||
|
const showErrorModal = ref(false)
|
||||||
|
const errorModalRow = ref(null)
|
||||||
|
|
||||||
|
function openError(row) {
|
||||||
|
errorModalRow.value = row
|
||||||
|
showErrorModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
const filterQueue = ref(null)
|
||||||
|
const filterStatus = ref(null)
|
||||||
|
const filterErrorType = ref(null)
|
||||||
|
|
||||||
|
const queueOptions = [
|
||||||
|
{ title: 'All queues', value: null },
|
||||||
|
{ title: 'import', value: 'import' },
|
||||||
|
{ title: 'thumbnail', value: 'thumbnail' },
|
||||||
|
{ title: 'ml', value: 'ml' },
|
||||||
|
{ title: 'download', value: 'download' },
|
||||||
|
{ title: 'scan', value: 'scan' },
|
||||||
|
{ title: 'maintenance', value: 'maintenance' },
|
||||||
|
{ title: 'default', value: 'default' },
|
||||||
|
]
|
||||||
|
const statusOptions = [
|
||||||
|
{ title: 'All statuses', value: null },
|
||||||
|
{ title: 'Running', value: 'running' },
|
||||||
|
{ title: 'OK', value: 'ok' },
|
||||||
|
{ title: 'Error', value: 'error' },
|
||||||
|
{ title: 'Timeout', value: 'timeout' },
|
||||||
|
{ title: 'Retry', value: 'retry' },
|
||||||
|
]
|
||||||
|
|
||||||
|
let queuesPollId = null
|
||||||
|
let failuresPollId = null
|
||||||
|
|
||||||
|
function pollQueues() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadQueues()
|
||||||
|
store.loadWorkers()
|
||||||
|
store.loadRecentMinute()
|
||||||
|
}
|
||||||
|
function pollFailures() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadFailures()
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
pollQueues()
|
||||||
|
pollFailures()
|
||||||
|
store.setFilter({ queue: null, status: null })
|
||||||
|
store.loadRuns({ reset: true })
|
||||||
|
queuesPollId = setInterval(pollQueues, 3000)
|
||||||
|
failuresPollId = setInterval(pollFailures, 15000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (queuesPollId) clearInterval(queuesPollId)
|
||||||
|
if (failuresPollId) clearInterval(failuresPollId)
|
||||||
|
})
|
||||||
|
|
||||||
|
const errorTypes = computed(() => {
|
||||||
|
const counts = store.failures?.count_by_type || {}
|
||||||
|
return Object.entries(counts).map(([name, count]) => ({ name, count }))
|
||||||
|
})
|
||||||
|
|
||||||
|
const failureCount = computed(() => (store.failures?.recent || []).length)
|
||||||
|
|
||||||
|
const filteredFailures = computed(() => {
|
||||||
|
const all = store.failures?.recent || []
|
||||||
|
if (!filterErrorType.value) return all
|
||||||
|
return all.filter(r => r.error_type === filterErrorType.value)
|
||||||
|
})
|
||||||
|
|
||||||
|
function toggleErrorTypeFilter(name) {
|
||||||
|
filterErrorType.value = filterErrorType.value === name ? null : name
|
||||||
|
if (filterErrorType.value) {
|
||||||
|
// Mirror into the All-activity table too.
|
||||||
|
filterStatus.value = 'error'
|
||||||
|
onFilterChange()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onFilterChange() {
|
||||||
|
store.setFilter({ queue: filterQueue.value, status: filterStatus.value })
|
||||||
|
store.loadRuns({ reset: true })
|
||||||
|
}
|
||||||
|
function onLoadMore() { store.loadRuns({ reset: false }) }
|
||||||
|
function onRefresh() { store.loadRuns({ reset: true }) }
|
||||||
|
|
||||||
|
function statusIcon(status) {
|
||||||
|
return {
|
||||||
|
ok: 'mdi-check-circle',
|
||||||
|
error: 'mdi-close-circle',
|
||||||
|
timeout: 'mdi-clock-alert',
|
||||||
|
retry: 'mdi-refresh',
|
||||||
|
running: 'mdi-timer-sand',
|
||||||
|
}[status] || 'mdi-help-circle'
|
||||||
|
}
|
||||||
|
function statusColor(status) {
|
||||||
|
return {
|
||||||
|
ok: 'success', error: 'error', timeout: 'warning',
|
||||||
|
retry: 'info', running: 'accent',
|
||||||
|
}[status] || 'on-surface-variant'
|
||||||
|
}
|
||||||
|
function shortTaskName(name) {
|
||||||
|
if (!name) return ''
|
||||||
|
const parts = name.split('.')
|
||||||
|
return parts.slice(-1)[0]
|
||||||
|
}
|
||||||
|
function formatDuration(ms) {
|
||||||
|
if (ms == null) return '—'
|
||||||
|
if (ms < 1000) return `${ms} ms`
|
||||||
|
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
|
||||||
|
return `${(ms / 60_000).toFixed(1)} min`
|
||||||
|
}
|
||||||
|
function formatRelative(iso) {
|
||||||
|
if (!iso) return '—'
|
||||||
|
const then = new Date(iso).getTime()
|
||||||
|
const now = Date.now()
|
||||||
|
const diff = Math.max(0, (now - then) / 1000)
|
||||||
|
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||||
|
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||||
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||||
|
return `${Math.floor(diff / 86400)}d ago`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-activity { padding: 0; }
|
||||||
|
.fc-pills {
|
||||||
|
display: flex; gap: 6px; flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
.fc-tabular {
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
font-feature-settings: 'tnum';
|
||||||
|
}
|
||||||
|
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||||
|
.fc-err-link {
|
||||||
|
/* Styled as a text-only button so the error_type cell stays
|
||||||
|
visually identical to the prior tooltip-bearing row, but is
|
||||||
|
now a real clickable target with hover affordance. */
|
||||||
|
color: rgb(var(--v-theme-error, 220 80 80));
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
font: inherit;
|
||||||
|
text-decoration: underline dotted;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-err-link:hover { text-decoration: underline; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-tag-maint">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-tag-remove" size="small" />
|
||||||
|
<span>Tag maintenance</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
|
Remove tags with zero image associations and zero series-page
|
||||||
|
references. Auto-created tag rows that never got applied get
|
||||||
|
swept here.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="store.lastError"
|
||||||
|
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>{{ store.lastError }}</v-alert>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify"
|
||||||
|
:loading="loadingPreview"
|
||||||
|
class="mb-3"
|
||||||
|
@click="onPreview"
|
||||||
|
>Preview unused tags</v-btn>
|
||||||
|
|
||||||
|
<div v-if="preview">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
<strong>{{ preview.count }}</strong> unused tag(s).
|
||||||
|
<span v-if="preview.count > 50" class="fc-muted">
|
||||||
|
Showing first 50 names.
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
<div v-if="preview.sample_names?.length" class="fc-name-grid mb-3">
|
||||||
|
<span v-for="n in preview.sample_names" :key="n" class="fc-name">
|
||||||
|
{{ n }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<v-btn
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete-sweep"
|
||||||
|
:disabled="!preview.count"
|
||||||
|
:loading="committing"
|
||||||
|
@click="onCommit"
|
||||||
|
>Delete {{ preview.count }} unused tag(s)</v-btn>
|
||||||
|
</div>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useAdminStore } from '../../stores/admin.js'
|
||||||
|
|
||||||
|
const store = useAdminStore()
|
||||||
|
const preview = ref(null)
|
||||||
|
const loadingPreview = ref(false)
|
||||||
|
const committing = ref(false)
|
||||||
|
|
||||||
|
async function onPreview() {
|
||||||
|
loadingPreview.value = true
|
||||||
|
try {
|
||||||
|
preview.value = await store.pruneUnusedTags({ dryRun: true })
|
||||||
|
} finally {
|
||||||
|
loadingPreview.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCommit() {
|
||||||
|
committing.value = true
|
||||||
|
try {
|
||||||
|
const result = await store.pruneUnusedTags({ dryRun: false })
|
||||||
|
preview.value = { count: 0, sample_names: result.sample_names || [] }
|
||||||
|
} finally {
|
||||||
|
committing.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-tag-maint { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-name-grid {
|
||||||
|
display: flex; flex-wrap: wrap; gap: 4px 8px;
|
||||||
|
max-height: 200px; overflow-y: auto;
|
||||||
|
padding: 8px; border-radius: 4px;
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
}
|
||||||
|
.fc-name {
|
||||||
|
font-family: 'JetBrains Mono', monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
<template>
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Thumbnail backfill</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="text-body-2 mb-3">
|
||||||
|
Scan the library for images with no thumbnail, or whose thumbnail file
|
||||||
|
is missing or corrupt on disk. Repair candidates are re-enqueued for
|
||||||
|
thumbnail generation. Safe to re-run.
|
||||||
|
</p>
|
||||||
|
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||||
|
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
||||||
|
</v-btn>
|
||||||
|
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
||||||
|
const store = useThumbnailsStore()
|
||||||
|
const busy = ref(false)
|
||||||
|
const done = ref(false)
|
||||||
|
async function run () {
|
||||||
|
busy.value = true
|
||||||
|
try { await store.triggerBackfill(); done.value = true }
|
||||||
|
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
||||||
|
finally { busy.value = false }
|
||||||
|
}
|
||||||
|
</script>
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
export const useAdminStore = defineStore('admin', () => {
|
||||||
|
const api = useApi()
|
||||||
|
const lastError = ref(null)
|
||||||
|
|
||||||
|
// --- Tier-C: artist cascade ---------------------------------------
|
||||||
|
|
||||||
|
async function projectArtistCascade(slug) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
`/api/admin/artists/${encodeURIComponent(slug)}/cascade-delete`,
|
||||||
|
{ body: { dry_run: true } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function dispatchArtistCascade(slug, confirm) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
`/api/admin/artists/${encodeURIComponent(slug)}/cascade-delete`,
|
||||||
|
{ body: { dry_run: false, confirm } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Tier-C: bulk image delete ------------------------------------
|
||||||
|
|
||||||
|
async function projectBulkImageDelete(imageIds) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
'/api/admin/images/bulk-delete',
|
||||||
|
{ body: { image_ids: imageIds, dry_run: true } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function dispatchBulkImageDelete(imageIds, confirm) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
'/api/admin/images/bulk-delete',
|
||||||
|
{ body: { image_ids: imageIds, dry_run: false, confirm } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Tier-B: tag delete + merge -----------------------------------
|
||||||
|
|
||||||
|
async function deleteTag(tagId) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.delete(`/api/admin/tags/${tagId}`)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mergeTags(destId, sourceId) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
`/api/admin/tags/${destId}/merge`,
|
||||||
|
{ body: { source_id: sourceId } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function tagUsageCount(tagId) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
const body = await api.get(`/api/admin/tags/${tagId}/usage-count`)
|
||||||
|
return body.count
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Tier-A: prune unused -----------------------------------------
|
||||||
|
|
||||||
|
async function pruneUnusedTags({ dryRun = true } = {}) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
'/api/admin/tags/prune-unused',
|
||||||
|
{ body: { dry_run: dryRun } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Polls /api/system/activity/runs?queue=maintenance every 3s,
|
||||||
|
* resolves when a task_run row with the given celery task_id
|
||||||
|
* reaches a terminal status (ok / error / timeout). Returns the
|
||||||
|
* row. Times out after 30 min by default.
|
||||||
|
*/
|
||||||
|
async function pollTaskUntilDone(taskId, { timeoutMs = 1_800_000 } = {}) {
|
||||||
|
const deadline = Date.now() + timeoutMs
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const body = await api.get(
|
||||||
|
'/api/system/activity/runs',
|
||||||
|
{ params: { queue: 'maintenance', limit: 20 } },
|
||||||
|
)
|
||||||
|
const row = (body.runs || []).find(r => r.celery_task_id === taskId)
|
||||||
|
if (row && ['ok', 'error', 'timeout'].includes(row.status)) {
|
||||||
|
return row
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 3000))
|
||||||
|
}
|
||||||
|
throw new Error(`task ${taskId} did not finish within ${timeoutMs}ms`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
lastError,
|
||||||
|
projectArtistCascade,
|
||||||
|
dispatchArtistCascade,
|
||||||
|
projectBulkImageDelete,
|
||||||
|
dispatchBulkImageDelete,
|
||||||
|
deleteTag,
|
||||||
|
mergeTags,
|
||||||
|
tagUsageCount,
|
||||||
|
pruneUnusedTags,
|
||||||
|
pollTaskUntilDone,
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { defineStore } from 'pinia'
|
import { defineStore } from 'pinia'
|
||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
import { usePostsStore } from './posts.js'
|
||||||
|
|
||||||
const PAGE = 60
|
const PAGE = 60
|
||||||
|
|
||||||
@@ -15,7 +16,11 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
const notFound = ref(false)
|
const notFound = ref(false)
|
||||||
let started = false
|
let started = false
|
||||||
|
|
||||||
async function load(slug) {
|
async function load (slug) {
|
||||||
|
// Cross-artist reset: clear this store AND the posts store so the new
|
||||||
|
// artist doesn't briefly render with the previous artist's content
|
||||||
|
// when the user is on the Posts tab. (Gallery tab uses this artist
|
||||||
|
// store's own images list — cleared above.)
|
||||||
overview.value = null
|
overview.value = null
|
||||||
images.value = []
|
images.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
@@ -23,6 +28,7 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
started = false
|
started = false
|
||||||
error.value = null
|
error.value = null
|
||||||
loading.value = true
|
loading.value = true
|
||||||
|
usePostsStore().$reset?.()
|
||||||
try {
|
try {
|
||||||
overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`)
|
overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`)
|
||||||
await loadMoreImages(slug)
|
await loadMoreImages(slug)
|
||||||
@@ -34,7 +40,7 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadMoreImages(slug) {
|
async function loadMoreImages (slug) {
|
||||||
if (imagesLoading.value) return
|
if (imagesLoading.value) return
|
||||||
if (started && nextCursor.value === null) return
|
if (started && nextCursor.value === null) return
|
||||||
imagesLoading.value = true
|
imagesLoading.value = true
|
||||||
@@ -55,9 +61,13 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const hasMoreImages = computed(() => !started || nextCursor.value !== null)
|
const hasMoreImages = computed(() => !started || nextCursor.value !== null)
|
||||||
|
const postCount = computed(() => overview.value?.post_count ?? null)
|
||||||
|
const imageCount = computed(() => overview.value?.image_count ?? null)
|
||||||
|
const lastAdded = computed(() => overview.value?.date_range?.max ?? null)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
overview, images, loading, imagesLoading, error, notFound,
|
overview, images, loading, imagesLoading, error, notFound,
|
||||||
hasMoreImages, load, loadMoreImages
|
hasMoreImages, postCount, imageCount, lastAdded,
|
||||||
|
load, loadMoreImages,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
export const useBackupStore = defineStore('backup', () => {
|
||||||
|
const api = useApi()
|
||||||
|
|
||||||
|
const dbRuns = ref([])
|
||||||
|
const imagesRuns = ref([])
|
||||||
|
const settings = ref(null)
|
||||||
|
const loading = ref({ dbRuns: false, imagesRuns: false, settings: false })
|
||||||
|
const lastError = ref(null)
|
||||||
|
|
||||||
|
async function loadRuns(kind) {
|
||||||
|
const targetRef = kind === 'db' ? dbRuns : imagesRuns
|
||||||
|
const loadingKey = kind === 'db' ? 'dbRuns' : 'imagesRuns'
|
||||||
|
loading.value[loadingKey] = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
const body = await api.get('/api/system/backup/runs', {
|
||||||
|
params: { kind, limit: 50 },
|
||||||
|
})
|
||||||
|
targetRef.value = body.runs || []
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value[loadingKey] = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function triggerBackup(kind, tag = null) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
await api.post(`/api/system/backup/${kind}`, {
|
||||||
|
body: tag ? { tag } : {},
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restore(runId, confirm) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
await api.post(`/api/system/backup/runs/${runId}/restore`, {
|
||||||
|
body: { confirm },
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteRun(runId, confirm) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
await api.delete(`/api/system/backup/runs/${runId}`, {
|
||||||
|
body: { confirm },
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setTag(runId, tag) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
await api.patch(`/api/system/backup/runs/${runId}`, {
|
||||||
|
body: { tag },
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadSettings() {
|
||||||
|
loading.value.settings = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
settings.value = await api.get('/api/system/backup/settings')
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value.settings = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function patchSettings(patch) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
settings.value = await api.patch('/api/system/backup/settings', {
|
||||||
|
body: patch,
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
dbRuns, imagesRuns, settings, loading, lastError,
|
||||||
|
loadRuns, triggerBackup, restore, deleteRun, setTag,
|
||||||
|
loadSettings, patchSettings,
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
export const useCleanupStore = defineStore('cleanup', () => {
|
||||||
|
const api = useApi()
|
||||||
|
|
||||||
|
// Defaults sourced from ImportSettings on mount. Cards pre-fill from
|
||||||
|
// these so the common case ("apply current import filters
|
||||||
|
// retroactively") is one click; operator can override per-audit.
|
||||||
|
const defaults = ref({
|
||||||
|
min_width: 0,
|
||||||
|
min_height: 0,
|
||||||
|
transparency_threshold: 0.9,
|
||||||
|
single_color_threshold: 0.95,
|
||||||
|
single_color_tolerance: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
const recentRuns = ref([])
|
||||||
|
|
||||||
|
async function loadDefaults() {
|
||||||
|
const s = await api.get('/api/settings/import')
|
||||||
|
defaults.value = {
|
||||||
|
min_width: s.min_width ?? 0,
|
||||||
|
min_height: s.min_height ?? 0,
|
||||||
|
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||||
|
single_color_threshold: s.single_color_threshold ?? 0.95,
|
||||||
|
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function previewMinDim(min_width, min_height) {
|
||||||
|
return await api.post('/api/cleanup/min-dimension/preview', {
|
||||||
|
body: { min_width, min_height },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteMinDim(min_width, min_height, confirm) {
|
||||||
|
return await api.post('/api/cleanup/min-dimension/delete', {
|
||||||
|
body: { min_width, min_height, confirm },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startAudit(rule, params) {
|
||||||
|
return await api.post('/api/cleanup/audit', { body: { rule, params } })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAudit(id) {
|
||||||
|
return await api.get(`/api/cleanup/audit/${id}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadHistory(limit = 20) {
|
||||||
|
const body = await api.get(`/api/cleanup/audit?limit=${limit}`)
|
||||||
|
recentRuns.value = body.runs
|
||||||
|
return body.runs
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyAudit(id, confirm) {
|
||||||
|
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cancelAudit(id) {
|
||||||
|
return await api.post(`/api/cleanup/audit/${id}/cancel`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
defaults, recentRuns,
|
||||||
|
loadDefaults,
|
||||||
|
previewMinDim, deleteMinDim,
|
||||||
|
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
|
||||||
|
}
|
||||||
|
})
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user