Compare commits
82 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b8dce6c483 | |||
| 832345a245 | |||
| a0136fa30d | |||
| de1a4b64b7 | |||
| 3f500e592e | |||
| d97e3f9b59 | |||
| 035c49f675 | |||
| e41ab1cca5 | |||
| 42c6b642c2 | |||
| ad3d34a1fc | |||
| b5289ed372 | |||
| f6aa805725 | |||
| f096c9a5fb | |||
| 676a86b514 | |||
| 44cc625d4a | |||
| f7ee122243 | |||
| 7c6f11964a | |||
| 94c60c0af2 | |||
| 6df102b83d | |||
| 2ae01d27e3 | |||
| 718cc79905 | |||
| e78a35d333 | |||
| 83bd3b4b2d | |||
| aecedd9fe4 | |||
| 1e34b1b428 | |||
| 102c21feaa | |||
| 57a338f7e6 | |||
| d04983138a | |||
| 9ec6fdb596 | |||
| 86ad9b80e9 | |||
| 2b05f147f4 | |||
| 70e1e010d1 | |||
| d3d4320ed5 | |||
| 7d42cddb11 | |||
| 1f01c4819a | |||
| 06d527cb92 | |||
| e9ea376aed | |||
| 882cb491ba | |||
| 319e7de547 | |||
| e43312a129 | |||
| 8f2732a56f | |||
| c3e855bd9b | |||
| d1c0b82a22 | |||
| 37fcc74954 | |||
| 6a532c1497 | |||
| 368613068a | |||
| ddbb84d8aa | |||
| 9b252948f9 | |||
| 89d0cb2124 | |||
| 36611cbe00 | |||
| 48ef22445a | |||
| e523d0ac94 | |||
| 541e2bfe6a | |||
| 7782672a51 | |||
| d12b51f6b7 | |||
| 0cda46fcdb | |||
| 79fee98db4 | |||
| b1d68929c5 | |||
| 553567738e | |||
| 5526b8dc78 | |||
| c9a3f12847 | |||
| 538c1591e8 | |||
| 16eb7075c4 | |||
| 4437488899 | |||
| 96039fc983 | |||
| 9f008af6c8 | |||
| be0f472894 | |||
| dd4874ae8d | |||
| 4ae9815d61 | |||
| 4da8538054 | |||
| d85c108cec | |||
| cd838ec904 | |||
| 2e1aaffd93 | |||
| 2065672a31 | |||
| 46d199450d | |||
| 379445c244 | |||
| b067a3eec1 | |||
| b7832c941d | |||
| df82abe75e | |||
| c1d3046778 | |||
| 9b01b19666 | |||
| a06c4f009f |
@@ -0,0 +1,59 @@
|
||||
name: extension
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
paths: ['extension/**']
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:22-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install web-ext
|
||||
run: cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
|
||||
sign-and-publish:
|
||||
needs: lint
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:22-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.RELEASE_TOKEN }}
|
||||
- name: Install web-ext + git
|
||||
run: |
|
||||
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
|
||||
cd extension && npm install --no-save --no-audit --no-fund
|
||||
- name: Sign XPI
|
||||
run: cd extension && npm run sign
|
||||
env:
|
||||
WEB_EXT_API_KEY: ${{ secrets.MOZILLA_AMO_JWT_KEY }}
|
||||
WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
|
||||
- name: Commit signed XPI to frontend/public/extension/
|
||||
run: |
|
||||
set -e
|
||||
XPI=$(ls extension/web-ext-artifacts/fabledcurator-*.xpi | head -1)
|
||||
if [ -z "$XPI" ]; then
|
||||
echo "No XPI produced by web-ext sign — exiting"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p frontend/public/extension
|
||||
cp "$XPI" frontend/public/extension/
|
||||
# Also copy as -latest.xpi so the FC server can serve a stable URL.
|
||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||
git config user.name "FC extension CI"
|
||||
git config user.email "noreply@fabledsword.com"
|
||||
git add frontend/public/extension/
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "ext: publish signed XPI $(basename $XPI)"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
@@ -0,0 +1,86 @@
|
||||
"""fc3i: task_run table
|
||||
|
||||
Revision ID: 0016
|
||||
Revises: 0015
|
||||
Create Date: 2026-05-24
|
||||
|
||||
Additive only. New table records every Celery task attempt via signal
|
||||
handlers (backend.app.celery_signals). Status is plain String(16) not
|
||||
Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard-
|
||||
fail at INSERT, String columns extend cleanly).
|
||||
|
||||
Composite indexes anticipate the three dashboard panes:
|
||||
- (queue, started_at desc) — per-lane recent activity
|
||||
- (status, started_at desc) — recent failures pane
|
||||
- (task_name, started_at desc) — drill-down by task
|
||||
|
||||
Indexed columns get individual indexes via `index=True` on the model;
|
||||
the composites below cover the multi-column lookups.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0016"
|
||||
down_revision: Union[str, None] = "0015"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"task_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("celery_task_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("queue", sa.String(length=32), nullable=False),
|
||||
sa.Column("task_name", sa.String(length=128), nullable=False),
|
||||
sa.Column("target_id", sa.Integer(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), nullable=False,
|
||||
server_default="running",
|
||||
),
|
||||
sa.Column("error_type", sa.String(length=128), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column("retry_count", sa.Integer(), nullable=True),
|
||||
sa.Column("worker_hostname", sa.String(length=128), nullable=True),
|
||||
sa.Column("args_summary", sa.String(length=255), nullable=True),
|
||||
)
|
||||
|
||||
# Single-column indexes (matches Mapped[...].index=True on model).
|
||||
op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"])
|
||||
op.create_index("ix_task_run_queue", "task_run", ["queue"])
|
||||
op.create_index("ix_task_run_task_name", "task_run", ["task_name"])
|
||||
op.create_index("ix_task_run_started_at", "task_run", ["started_at"])
|
||||
op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"])
|
||||
op.create_index("ix_task_run_status", "task_run", ["status"])
|
||||
|
||||
# Composite indexes for dashboard query patterns.
|
||||
op.create_index(
|
||||
"ix_task_run_queue_started",
|
||||
"task_run", ["queue", sa.text("started_at DESC")],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_task_run_status_started",
|
||||
"task_run", ["status", sa.text("started_at DESC")],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_task_run_name_started",
|
||||
"task_run", ["task_name", sa.text("started_at DESC")],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_task_run_name_started", table_name="task_run")
|
||||
op.drop_index("ix_task_run_status_started", table_name="task_run")
|
||||
op.drop_index("ix_task_run_queue_started", table_name="task_run")
|
||||
op.drop_index("ix_task_run_status", table_name="task_run")
|
||||
op.drop_index("ix_task_run_finished_at", table_name="task_run")
|
||||
op.drop_index("ix_task_run_started_at", table_name="task_run")
|
||||
op.drop_index("ix_task_run_task_name", table_name="task_run")
|
||||
op.drop_index("ix_task_run_queue", table_name="task_run")
|
||||
op.drop_index("ix_task_run_celery_task_id", table_name="task_run")
|
||||
op.drop_table("task_run")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""fc3h: backup_run table
|
||||
|
||||
Revision ID: 0017
|
||||
Revises: 0016
|
||||
Create Date: 2026-05-24
|
||||
|
||||
Additive. New table records every backup/restore attempt with artifact
|
||||
metadata. Lifecycle tracking lives in task_run from FC-3i; this is
|
||||
artifact-only (paths, sizes, tag, restore lineage).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0017"
|
||||
down_revision: Union[str, None] = "0016"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"backup_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), nullable=False,
|
||||
server_default="pending",
|
||||
),
|
||||
sa.Column("tag", sa.String(length=64), nullable=True),
|
||||
sa.Column("triggered_by", sa.String(length=32), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("sql_path", sa.Text(), nullable=True),
|
||||
sa.Column("tar_path", sa.Text(), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"manifest", sa.JSON(), nullable=False, server_default="{}",
|
||||
),
|
||||
sa.Column(
|
||||
"restored_from_id", sa.Integer(),
|
||||
sa.ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Single-column indexes (matches Mapped[...].index=True).
|
||||
op.create_index("ix_backup_run_kind", "backup_run", ["kind"])
|
||||
op.create_index("ix_backup_run_status", "backup_run", ["status"])
|
||||
op.create_index("ix_backup_run_tag", "backup_run", ["tag"])
|
||||
op.create_index("ix_backup_run_started_at", "backup_run", ["started_at"])
|
||||
op.create_index("ix_backup_run_finished_at", "backup_run", ["finished_at"])
|
||||
|
||||
# Composite indexes for dashboard query patterns.
|
||||
op.create_index(
|
||||
"ix_backup_run_kind_started",
|
||||
"backup_run", ["kind", sa.text("started_at DESC")],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_backup_run_status_finished",
|
||||
"backup_run", ["status", sa.text("finished_at DESC")],
|
||||
)
|
||||
# Partial index: only tagged rows participate in retention-exempt query.
|
||||
op.create_index(
|
||||
"ix_backup_run_tag_partial",
|
||||
"backup_run", ["tag"],
|
||||
postgresql_where=sa.text("tag IS NOT NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_backup_run_tag_partial", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_status_finished", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_kind_started", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_finished_at", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_started_at", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_tag", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_status", table_name="backup_run")
|
||||
op.drop_index("ix_backup_run_kind", table_name="backup_run")
|
||||
op.drop_table("backup_run")
|
||||
@@ -0,0 +1,62 @@
|
||||
"""fc3h: backup_* knobs on import_settings
|
||||
|
||||
Revision ID: 0018
|
||||
Revises: 0017
|
||||
Create Date: 2026-05-24
|
||||
|
||||
Adds four columns to the singleton import_settings row:
|
||||
- backup_db_nightly_enabled (default False — opt-in)
|
||||
- backup_db_nightly_hour_utc (default 3)
|
||||
- backup_db_keep_last_n (default 14)
|
||||
- backup_images_keep_last_n (default 3)
|
||||
|
||||
server_default ensures the singleton row is backfilled in place
|
||||
without an UPDATE statement.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0018"
|
||||
down_revision: Union[str, None] = "0017"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_db_nightly_enabled", sa.Boolean(),
|
||||
nullable=False, server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_db_nightly_hour_utc", sa.Integer(),
|
||||
nullable=False, server_default="3",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_db_keep_last_n", sa.Integer(),
|
||||
nullable=False, server_default="14",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_images_keep_last_n", sa.Integer(),
|
||||
nullable=False, server_default="3",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "backup_images_keep_last_n")
|
||||
op.drop_column("import_settings", "backup_db_keep_last_n")
|
||||
op.drop_column("import_settings", "backup_db_nightly_hour_utc")
|
||||
op.drop_column("import_settings", "backup_db_nightly_enabled")
|
||||
@@ -14,6 +14,7 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
|
||||
|
||||
|
||||
def all_blueprints() -> list[Blueprint]:
|
||||
from .admin import admin_bp
|
||||
from .aliases import aliases_bp
|
||||
from .allowlist import allowlist_bp
|
||||
from .artist import artist_bp
|
||||
@@ -21,6 +22,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .attachments import attachments_bp
|
||||
from .credentials import credentials_bp
|
||||
from .downloads import downloads_bp
|
||||
from .extension import extension_bp
|
||||
from .gallery import gallery_bp
|
||||
from .import_admin import import_admin_bp
|
||||
from .migrate import migrate_bp
|
||||
@@ -32,6 +34,8 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .showcase import showcase_bp
|
||||
from .sources import sources_bp
|
||||
from .suggestions import suggestions_bp
|
||||
from .system_activity import system_activity_bp
|
||||
from .system_backup import system_backup_bp
|
||||
from .tags import tags_bp
|
||||
return [
|
||||
api_bp,
|
||||
@@ -43,6 +47,9 @@ def all_blueprints() -> list[Blueprint]:
|
||||
artists_bp,
|
||||
showcase_bp,
|
||||
settings_bp,
|
||||
system_activity_bp,
|
||||
system_backup_bp,
|
||||
admin_bp,
|
||||
import_admin_bp,
|
||||
migrate_bp,
|
||||
suggestions_bp,
|
||||
@@ -53,5 +60,6 @@ def all_blueprints() -> list[Blueprint]:
|
||||
platforms_bp,
|
||||
posts_bp,
|
||||
credentials_bp,
|
||||
extension_bp,
|
||||
downloads_bp,
|
||||
]
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
"""FC-3k: /api/admin — destructive admin actions.
|
||||
|
||||
Five action surfaces:
|
||||
POST /api/admin/artists/<slug>/cascade-delete (Tier C)
|
||||
POST /api/admin/images/bulk-delete (Tier C)
|
||||
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||
POST /api/admin/tags/prune-unused (Tier A)
|
||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||
|
||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||
no dispatch) and a confirm body field (server-recomputed token).
|
||||
Long-running ops dispatch a maintenance-queue Celery task; the UI
|
||||
tails FC-3i's /api/system/activity/runs to surface progress.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist
|
||||
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
||||
|
||||
admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _bulk_image_confirm_token(image_ids: list[int]) -> str:
|
||||
"""Stable 8-hex token derived from the sorted id list. Mutates
|
||||
when the selection changes; stays the same across modal opens of
|
||||
the same selection so the operator can paste without confusion."""
|
||||
canon = ",".join(str(i) for i in sorted(image_ids))
|
||||
digest = hashlib.sha256(canon.encode("utf-8")).hexdigest()
|
||||
return digest[:8]
|
||||
|
||||
|
||||
@admin_bp.route("/artists/<slug>/cascade-delete", methods=["POST"])
|
||||
async def artist_cascade_delete(slug: str):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
supplied_confirm = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
artist = (await session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return _bad("not_found", status=404)
|
||||
artist_id = artist.id
|
||||
|
||||
projected = await session.run_sync(
|
||||
lambda sync_sess: project_artist_cascade(sync_sess, slug=slug)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return jsonify(projected)
|
||||
|
||||
expected = f"delete-artist-{artist_id}"
|
||||
if supplied_confirm != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
|
||||
from ..tasks.admin import delete_artist_cascade_task
|
||||
async_result = delete_artist_cascade_task.delay(artist_id=artist_id)
|
||||
return jsonify({"task_id": async_result.id}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/images/bulk-delete", methods=["POST"])
|
||||
async def images_bulk_delete():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
image_ids = body.get("image_ids")
|
||||
if not isinstance(image_ids, list) or not image_ids:
|
||||
return _bad("invalid_image_ids", detail="image_ids must be non-empty list of int")
|
||||
try:
|
||||
image_ids = [int(i) for i in image_ids]
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_image_ids", detail="image_ids must contain only ints")
|
||||
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
supplied_confirm = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
projected = await session.run_sync(
|
||||
lambda sync_sess: project_bulk_image_delete(
|
||||
sync_sess, image_ids=image_ids,
|
||||
)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return jsonify(projected)
|
||||
|
||||
sha8 = _bulk_image_confirm_token(image_ids)
|
||||
expected = f"delete-images-{sha8}"
|
||||
if supplied_confirm != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
|
||||
from ..tasks.admin import bulk_delete_images_task
|
||||
async_result = bulk_delete_images_task.delay(image_ids=image_ids)
|
||||
return jsonify({"task_id": async_result.id}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:tag_id>", methods=["DELETE"])
|
||||
async def tag_delete(tag_id: int):
|
||||
"""Tier-B sync delete. UI yes/no modal is the only confirmation."""
|
||||
from ..services.cleanup_service import delete_tag
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
result = await session.run_sync(
|
||||
lambda sync_sess: delete_tag(sync_sess, tag_id=tag_id)
|
||||
)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:dest_id>/merge", methods=["POST"])
|
||||
async def tag_merge(dest_id: int):
|
||||
"""Wraps TagService.merge. Source repoints to dest, dest survives,
|
||||
source row deleted, protective alias auto-created if source was
|
||||
ML-applied or allowlisted."""
|
||||
from ..services.tag_service import TagMergeConflict, TagService, TagValidationError
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
source_id = body.get("source_id")
|
||||
if not isinstance(source_id, int) or source_id == dest_id:
|
||||
return _bad("invalid_source_id", detail="source_id must be int and differ from dest")
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
result = await TagService(session).merge(
|
||||
source_id=source_id, target_id=dest_id,
|
||||
)
|
||||
except TagMergeConflict as exc:
|
||||
return _bad("merge_conflict", status=409, detail=str(exc))
|
||||
except TagValidationError as exc:
|
||||
return _bad("tag_kind_mismatch", detail=str(exc))
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
|
||||
# MergeResult is a frozen dataclass — flatten to dict.
|
||||
return jsonify({
|
||||
"result": {
|
||||
"target_id": result.target_id,
|
||||
"target_name": result.target_name,
|
||||
"target_kind": result.target_kind,
|
||||
"merged_count": result.merged_count,
|
||||
"alias_created": result.alias_created,
|
||||
"source_deleted": result.source_deleted,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:tag_id>/usage-count", methods=["GET"])
|
||||
async def tag_usage_count(tag_id: int):
|
||||
"""Helper for the Tier-B yes/no prompt; surfaces "N associations"
|
||||
in the dialog so the operator knows what they're nuking."""
|
||||
from ..services.cleanup_service import count_tag_associations
|
||||
|
||||
async with get_session() as session:
|
||||
count = await session.run_sync(
|
||||
lambda sync_sess: count_tag_associations(
|
||||
sync_sess, tag_id=tag_id,
|
||||
)
|
||||
)
|
||||
return jsonify({"count": count})
|
||||
|
||||
|
||||
@admin_bp.route("/tags/prune-unused", methods=["POST"])
|
||||
async def tags_prune_unused():
|
||||
"""Tier-A: dry-run preview list IS the prompt. UI calls with
|
||||
dry_run=true first, shows the list, operator clicks button to
|
||||
re-call with dry_run=false."""
|
||||
from ..services.cleanup_service import prune_unused_tags
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda sync_sess: prune_unused_tags(
|
||||
sync_sess, dry_run=dry_run,
|
||||
)
|
||||
)
|
||||
return jsonify(result)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""FC-3g: /api/extension — quick-add-source for the Firefox extension
|
||||
+ install-time manifest for the Settings card.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
UnknownPlatformError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
|
||||
extension_bp = Blueprint("extension", __name__, url_prefix="/api/extension")
|
||||
|
||||
# Default XPI directory; tests override via monkeypatching this module-
|
||||
# level constant.
|
||||
XPI_DIR = Path("/app/frontend/dist/extension")
|
||||
|
||||
_XPI_VERSION_RE = re.compile(r"fabledcurator-(?P<version>[\w.-]+)\.xpi$")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
async def _ext_key_required(session) -> bool:
|
||||
"""Unlike /api/credentials (which accepts the browser path with no
|
||||
header), quick-add-source writes server state and must be explicitly
|
||||
authenticated."""
|
||||
supplied = request.headers.get("X-Extension-Key")
|
||||
if supplied is None:
|
||||
return False
|
||||
stored = (await session.execute(
|
||||
select(AppSetting.value).where(AppSetting.key == "extension_api_key")
|
||||
)).scalar_one_or_none()
|
||||
return stored is not None and supplied == stored
|
||||
|
||||
|
||||
def _extract_version(xpi_name: str) -> str:
|
||||
m = _XPI_VERSION_RE.search(xpi_name)
|
||||
return m.group("version") if m else "unknown"
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(65536), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@extension_bp.route("/quick-add-source", methods=["POST"])
|
||||
async def quick_add_source():
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
url = body.get("url")
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session).quick_add_source(url)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
detail=str(exc),
|
||||
known=sorted(KNOWN_PLATFORMS),
|
||||
)
|
||||
except InvalidUrlError as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
return jsonify(result), (201 if result["created_source"] else 200)
|
||||
|
||||
|
||||
def _read_manifest_sync() -> dict | None:
|
||||
"""All the filesystem-touching work for /api/extension/manifest,
|
||||
in a sync helper so the async route can dispatch it via
|
||||
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
||||
if not XPI_DIR.is_dir():
|
||||
return None
|
||||
xpis = sorted(XPI_DIR.glob("fabledcurator-*.xpi"), key=lambda p: p.stat().st_mtime)
|
||||
if not xpis:
|
||||
return None
|
||||
latest = xpis[-1]
|
||||
return {
|
||||
"installed": True,
|
||||
"version": _extract_version(latest.name),
|
||||
"xpi_url": f"/extension/{latest.name}",
|
||||
"latest_url": "/extension/fabledcurator-latest.xpi",
|
||||
"sha256": _sha256(latest),
|
||||
}
|
||||
|
||||
|
||||
@extension_bp.route("/manifest", methods=["GET"])
|
||||
async def extension_manifest():
|
||||
info = await asyncio.to_thread(_read_manifest_sync)
|
||||
if info is None:
|
||||
return jsonify({"installed": False}), 404
|
||||
return jsonify(info)
|
||||
@@ -1,14 +1,11 @@
|
||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
||||
|
||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
||||
`export_file` field. All other kinds accept JSON. Apply-without-backup
|
||||
guard rejects non-dry-run ingests unless a pre_migration-tagged backup
|
||||
exists in the last 24h (override with body.force=true).
|
||||
`export_file` field. All other kinds accept JSON. Backup + rollback
|
||||
were retired in FC-3h (2026-05-24); use /api/system/backup/* instead.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
@@ -19,12 +16,12 @@ from ..tasks.migration import run_migration
|
||||
|
||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
||||
|
||||
# 'backup' + 'rollback' retired 2026-05-24 (FC-3h); see /api/system/backup/*.
|
||||
_VALID_KINDS = frozenset({
|
||||
"backup", "gs_ingest", "ir_ingest", "tag_apply",
|
||||
"ml_queue", "verify", "rollback", "cleanup",
|
||||
"gs_ingest", "ir_ingest", "tag_apply",
|
||||
"ml_queue", "verify", "cleanup",
|
||||
})
|
||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
@@ -33,19 +30,6 @@ def _bad(error: str, *, status: int = 400, **extra):
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _has_recent_pre_migration_backup() -> bool:
|
||||
from ..services.migrators import backup as backup_mod
|
||||
images_root = Path("/images")
|
||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
||||
if manifest is None:
|
||||
return False
|
||||
created_at_str = manifest.get("created_at")
|
||||
if not created_at_str:
|
||||
return False
|
||||
created_at = datetime.fromisoformat(created_at_str)
|
||||
return (datetime.now(UTC) - created_at) < timedelta(hours=24)
|
||||
|
||||
|
||||
def _run_to_dict(run: MigrationRun) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
@@ -78,7 +62,6 @@ async def create_run(kind: str):
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
return _bad("invalid_export_file", detail=str(exc))
|
||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
||||
force = str(form.get("force", "false")).lower() in ("true", "1", "yes")
|
||||
params: dict = {"data": data, "dry_run": dry_run}
|
||||
else:
|
||||
body = await request.get_json()
|
||||
@@ -87,20 +70,8 @@ async def create_run(kind: str):
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body")
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
force = bool(body.get("force", False))
|
||||
params = dict(body)
|
||||
|
||||
is_apply = (kind in _APPLY_KINDS) and not dry_run
|
||||
if is_apply and not force and not _has_recent_pre_migration_backup():
|
||||
return _bad(
|
||||
"no_backup",
|
||||
detail="apply action requires a pre_migration-tagged backup "
|
||||
"in the last 24h (or force=true).",
|
||||
)
|
||||
|
||||
if kind == "backup":
|
||||
params.setdefault("tag", "pre_migration")
|
||||
|
||||
async with get_session() as session:
|
||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
||||
session.add(run)
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""FC-3i: system activity dashboard endpoints.
|
||||
|
||||
Read-only. Combines Redis-broker queue depths (LLEN per queue),
|
||||
Celery worker introspection (celery inspect), and the task_run DB
|
||||
history into the surfaces the SystemActivityTab UI consumes.
|
||||
|
||||
All filesystem/sync-client work goes through asyncio.to_thread per
|
||||
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import desc, func, select
|
||||
|
||||
from ..config import get_config
|
||||
from ..extensions import get_session
|
||||
from ..models import TaskRun
|
||||
|
||||
system_activity_bp = Blueprint(
|
||||
"system_activity", __name__, url_prefix="/api/system/activity",
|
||||
)
|
||||
|
||||
# Canonical queue order — must match celery_app.task_routes. UI renders
|
||||
# in this order; queues with no LLEN response show as null rather than
|
||||
# absent.
|
||||
_QUEUE_NAMES = (
|
||||
"default", "import", "thumbnail", "ml",
|
||||
"download", "scan", "maintenance",
|
||||
)
|
||||
|
||||
# Cache module-level so all requests share the cache between polls.
|
||||
# Tests can reset via direct dict mutation if needed.
|
||||
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
|
||||
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
|
||||
_QUEUE_CACHE_TTL = 2.0
|
||||
_WORKER_CACHE_TTL = 5.0
|
||||
|
||||
|
||||
def _read_queues_sync() -> dict:
|
||||
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
|
||||
asyncio.to_thread. Per-queue try/except returns None on failure so
|
||||
one bad queue doesn't break the whole response."""
|
||||
import redis # local import; only this endpoint needs it
|
||||
|
||||
cfg = get_config()
|
||||
client = redis.Redis.from_url(cfg.celery_broker_url)
|
||||
out: dict = {}
|
||||
for name in _QUEUE_NAMES:
|
||||
try:
|
||||
out[name] = int(client.llen(name))
|
||||
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
|
||||
out[name] = None
|
||||
return {
|
||||
"queues": out,
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _read_workers_sync() -> dict:
|
||||
"""celery inspect active_queues + active. Returns per-worker info."""
|
||||
from ..celery_app import celery as celery_app
|
||||
|
||||
insp = celery_app.control.inspect(timeout=2.0)
|
||||
active_queues = insp.active_queues() or {}
|
||||
active_tasks = insp.active() or {}
|
||||
|
||||
workers: dict = {}
|
||||
for hostname, queues in active_queues.items():
|
||||
workers[hostname] = {
|
||||
"queues": sorted({q["name"] for q in queues}),
|
||||
"active_count": len(active_tasks.get(hostname, [])),
|
||||
}
|
||||
return {
|
||||
"workers": workers,
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@system_activity_bp.route("/queues", methods=["GET"])
|
||||
async def get_queues():
|
||||
"""Per-queue Redis LLEN. Cached 2s.
|
||||
|
||||
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
||||
"""
|
||||
now = time.time()
|
||||
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
||||
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
||||
_QUEUE_CACHE["ts"] = now
|
||||
return jsonify(_QUEUE_CACHE["data"])
|
||||
|
||||
|
||||
@system_activity_bp.route("/workers", methods=["GET"])
|
||||
async def get_workers():
|
||||
"""Live celery inspect. Cached 5s.
|
||||
|
||||
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
|
||||
"""
|
||||
now = time.time()
|
||||
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
|
||||
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
|
||||
_WORKER_CACHE["ts"] = now
|
||||
return jsonify(_WORKER_CACHE["data"])
|
||||
|
||||
|
||||
@system_activity_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
"""Paginated task_run history. Query params:
|
||||
queue=<name> filter to one queue
|
||||
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||
limit=<int> default 50, max 200
|
||||
before_id=<int> cursor for keyset pagination
|
||||
|
||||
Response: {runs: [...], next_cursor: id|null}
|
||||
"""
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
if limit < 1:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
|
||||
queue = request.args.get("queue")
|
||||
status = request.args.get("status")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(TaskRun).order_by(desc(TaskRun.id))
|
||||
if queue:
|
||||
stmt = stmt.where(TaskRun.queue == queue)
|
||||
if status:
|
||||
stmt = stmt.where(TaskRun.status == status)
|
||||
if before_id is not None:
|
||||
stmt = stmt.where(TaskRun.id < before_id)
|
||||
stmt = stmt.limit(limit + 1)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return jsonify({
|
||||
"runs": [_row_to_dict(r) for r in rows],
|
||||
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||
})
|
||||
|
||||
|
||||
@system_activity_bp.route("/failures", methods=["GET"])
|
||||
async def list_failures():
|
||||
"""Recent failures across all lanes (24h window).
|
||||
|
||||
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
|
||||
"""
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
|
||||
since = datetime.now(UTC) - timedelta(hours=24)
|
||||
|
||||
async with get_session() as session:
|
||||
recent_stmt = (
|
||||
select(TaskRun)
|
||||
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||
.where(TaskRun.finished_at >= since)
|
||||
.order_by(desc(TaskRun.finished_at))
|
||||
.limit(limit)
|
||||
)
|
||||
recent = (await session.execute(recent_stmt)).scalars().all()
|
||||
|
||||
count_stmt = (
|
||||
select(TaskRun.error_type, func.count(TaskRun.id))
|
||||
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||
.where(TaskRun.finished_at >= since)
|
||||
.group_by(TaskRun.error_type)
|
||||
.order_by(desc(func.count(TaskRun.id)))
|
||||
)
|
||||
counts = (await session.execute(count_stmt)).all()
|
||||
|
||||
return jsonify({
|
||||
"recent": [_row_to_dict(r) for r in recent],
|
||||
"count_by_type": {
|
||||
(row[0] or "Unknown"): row[1]
|
||||
for row in counts
|
||||
},
|
||||
"since": since.isoformat(),
|
||||
})
|
||||
|
||||
|
||||
def _row_to_dict(r: TaskRun) -> dict:
|
||||
return {
|
||||
"id": r.id,
|
||||
"queue": r.queue,
|
||||
"task_name": r.task_name,
|
||||
"target_id": r.target_id,
|
||||
"celery_task_id": r.celery_task_id,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"duration_ms": r.duration_ms,
|
||||
"status": r.status,
|
||||
"error_type": r.error_type,
|
||||
"error_message": r.error_message,
|
||||
"retry_count": r.retry_count,
|
||||
"worker_hostname": r.worker_hostname,
|
||||
"args_summary": r.args_summary,
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
"""FC-3h: /api/system/backup — create/list/restore/delete/tag for
|
||||
DB + image backups.
|
||||
|
||||
Read endpoints are public on FC (operator-facing internal API; same
|
||||
posture as /api/system/activity). Write endpoints take a typed
|
||||
`confirm` body field that must match a server-generated token for
|
||||
that backup row, to prevent click-to-destroy by stale browser tabs
|
||||
or accidental cURL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import desc, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import BackupRun, ImportSettings
|
||||
|
||||
system_backup_bp = Blueprint(
|
||||
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||
)
|
||||
|
||||
_KINDS = frozenset({"db", "images"})
|
||||
_TAG_MAX_LEN = 64
|
||||
_BACKUP_SETTINGS_FIELDS = (
|
||||
"backup_db_nightly_enabled",
|
||||
"backup_db_nightly_hour_utc",
|
||||
"backup_db_keep_last_n",
|
||||
"backup_images_keep_last_n",
|
||||
)
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _row_to_dict(r: BackupRun) -> dict:
|
||||
return {
|
||||
"id": r.id,
|
||||
"kind": r.kind,
|
||||
"status": r.status,
|
||||
"tag": r.tag,
|
||||
"triggered_by": r.triggered_by,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"duration_seconds": (
|
||||
int((r.finished_at - r.started_at).total_seconds())
|
||||
if r.finished_at and r.started_at else None
|
||||
),
|
||||
"sql_path": r.sql_path,
|
||||
"tar_path": r.tar_path,
|
||||
"size_bytes": r.size_bytes,
|
||||
"error": r.error,
|
||||
"restored_from_id": r.restored_from_id,
|
||||
"manifest": r.manifest or {},
|
||||
}
|
||||
|
||||
|
||||
def _validate_tag(tag):
|
||||
if tag is None:
|
||||
return None
|
||||
if not isinstance(tag, str):
|
||||
return _bad("invalid_tag", detail="tag must be string or null")
|
||||
tag = tag.strip()
|
||||
if not tag:
|
||||
return None
|
||||
if len(tag) > _TAG_MAX_LEN:
|
||||
return _bad("invalid_tag", detail=f"tag too long (max {_TAG_MAX_LEN})")
|
||||
return tag
|
||||
|
||||
|
||||
def _validate_backup_settings_patch(body: dict):
|
||||
if "backup_db_nightly_enabled" in body and not isinstance(
|
||||
body["backup_db_nightly_enabled"], bool,
|
||||
):
|
||||
return _bad("invalid_value", detail="backup_db_nightly_enabled must be bool")
|
||||
if "backup_db_nightly_hour_utc" in body:
|
||||
v = body["backup_db_nightly_hour_utc"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (0 <= v <= 23):
|
||||
return _bad("invalid_value", detail="backup_db_nightly_hour_utc must be 0..23")
|
||||
if "backup_db_keep_last_n" in body:
|
||||
v = body["backup_db_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
|
||||
return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
|
||||
if "backup_images_keep_last_n" in body:
|
||||
v = body["backup_images_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 100):
|
||||
return _bad("invalid_value", detail="backup_images_keep_last_n must be 1..100")
|
||||
return None
|
||||
|
||||
|
||||
@system_backup_bp.route("/db", methods=["POST"])
|
||||
async def trigger_db_backup():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
tag = _validate_tag(body.get("tag"))
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
from ..tasks.backup import backup_db_task
|
||||
backup_db_task.delay(tag=tag, triggered_by="manual")
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/images", methods=["POST"])
|
||||
async def trigger_images_backup():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
tag = _validate_tag(body.get("tag"))
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
from ..tasks.backup import backup_images_task
|
||||
backup_images_task.delay(tag=tag, triggered_by="manual")
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
if limit < 1:
|
||||
return _bad("invalid_limit")
|
||||
kind = request.args.get("kind")
|
||||
if kind is not None and kind not in _KINDS:
|
||||
return _bad("invalid_kind", detail=f"kind must be one of {sorted(_KINDS)}")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(BackupRun).order_by(desc(BackupRun.id))
|
||||
if kind:
|
||||
stmt = stmt.where(BackupRun.kind == kind)
|
||||
if before_id is not None:
|
||||
stmt = stmt.where(BackupRun.id < before_id)
|
||||
stmt = stmt.limit(limit + 1)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return jsonify({
|
||||
"runs": [_row_to_dict(r) for r in rows],
|
||||
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||
})
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["GET"])
|
||||
async def get_run(run_id: int):
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_row_to_dict(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["PATCH"])
|
||||
async def patch_run(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
if "tag" not in body:
|
||||
return _bad("invalid_body", detail="tag required")
|
||||
tag = _validate_tag(body["tag"])
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
row.tag = tag
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return jsonify(_row_to_dict(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>/restore", methods=["POST"])
|
||||
async def trigger_restore(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
supplied = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
if row.status != "ok":
|
||||
return _bad(
|
||||
"not_restorable",
|
||||
detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
|
||||
)
|
||||
expected = f"restore-{row.kind}-{row.id}"
|
||||
if supplied != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
kind = row.kind
|
||||
|
||||
if kind == "db":
|
||||
from ..tasks.backup import restore_db_task
|
||||
restore_db_task.delay(source_backup_run_id=run_id)
|
||||
else: # 'images' (the only other value _KINDS allows via the trigger path)
|
||||
from ..tasks.backup import restore_images_task
|
||||
restore_images_task.delay(source_backup_run_id=run_id)
|
||||
return jsonify({"status": "dispatched", "kind": kind}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
|
||||
async def delete_run(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
supplied = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
expected = f"delete-{row.kind}-{row.id}"
|
||||
if supplied != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
from ..services import backup_service
|
||||
backup_service.unlink_artifact_files(
|
||||
sql_path=row.sql_path, tar_path=row.tar_path,
|
||||
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||
)
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@system_backup_bp.route("/settings", methods=["GET"])
|
||||
async def get_settings():
|
||||
async with get_session() as session:
|
||||
row = (await session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
)).scalar_one()
|
||||
return jsonify({
|
||||
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
||||
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
||||
"backup_db_keep_last_n": row.backup_db_keep_last_n,
|
||||
"backup_images_keep_last_n": row.backup_images_keep_last_n,
|
||||
})
|
||||
|
||||
|
||||
@system_backup_bp.route("/settings", methods=["PATCH"])
|
||||
async def patch_settings():
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
|
||||
err = _validate_backup_settings_patch(body)
|
||||
if err is not None:
|
||||
return err
|
||||
|
||||
async with get_session() as session:
|
||||
row = (await session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
)).scalar_one()
|
||||
for field in _BACKUP_SETTINGS_FIELDS:
|
||||
if field in body:
|
||||
setattr(row, field, body[field])
|
||||
await session.commit()
|
||||
return await get_settings()
|
||||
@@ -31,6 +31,8 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.migration",
|
||||
"backend.app.tasks.ml",
|
||||
"backend.app.tasks.download",
|
||||
"backend.app.tasks.backup",
|
||||
"backend.app.tasks.admin",
|
||||
],
|
||||
)
|
||||
app.conf.update(
|
||||
@@ -43,6 +45,8 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||
},
|
||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||
task_acks_late=True,
|
||||
@@ -81,9 +85,28 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
"recover-stalled-task-runs": {
|
||||
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
||||
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
||||
},
|
||||
"prune-task-runs": {
|
||||
"task": "backend.app.tasks.maintenance.prune_task_runs",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
"fc3h-backup-db-nightly": {
|
||||
"task": "backend.app.tasks.backup.backup_db_nightly",
|
||||
"schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour
|
||||
},
|
||||
"fc3h-prune-backups": {
|
||||
"task": "backend.app.tasks.backup.prune_backups",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
},
|
||||
timezone="UTC",
|
||||
)
|
||||
# FC-3i: register task_run signal handlers (side-effect import).
|
||||
from . import celery_signals # noqa: F401
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""FC-3i: task_run lifecycle via Celery signals.
|
||||
|
||||
Subscribes to task_prerun / task_postrun / task_failure / task_retry
|
||||
and persists one task_run row per task attempt. Drop-in for every
|
||||
existing and future Celery task — no per-task instrumentation.
|
||||
|
||||
Signal handlers run inside the worker process (sync context); DB
|
||||
writes go through the existing shared sync engine
|
||||
(backend.app.tasks._sync_engine.sync_session_factory) — one engine
|
||||
per worker process, not per-task, so we don't blow Postgres
|
||||
max_connections under load (the reason FC-3g shared-engine fix
|
||||
existed).
|
||||
|
||||
Failure-mode discipline (operator-pressed point): every handler is
|
||||
wrapped in try/except that swallows + logs. If the DB is down or the
|
||||
handler has a bug, the real task still runs — the dashboard goes
|
||||
dark for that interval. Monitoring NEVER breaks the thing it's
|
||||
monitoring.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
||||
|
||||
from .models import TaskRun
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Celery-internal tasks that would generate dashboard noise without
|
||||
# operational value. Conservative list; extend only when a specific
|
||||
# task proves noisy.
|
||||
_UNTRACKED_TASK_NAMES = frozenset({
|
||||
"celery.chord_unlock",
|
||||
"celery.backend_cleanup",
|
||||
"celery.chunks",
|
||||
})
|
||||
|
||||
_MAX_ERROR_MESSAGE_LEN = 2000
|
||||
_MAX_ARGS_SUMMARY_LEN = 255
|
||||
_MAX_WORKER_HOSTNAME_LEN = 128
|
||||
|
||||
# PostgreSQL Integer is signed 32-bit. Tasks called with a first-arg
|
||||
# int outside this range (e.g. an absurdly large mock value, or a
|
||||
# string-of-digits coercible to int but bigger than 2^31-1) would crash
|
||||
# the INSERT with NumericValueOutOfRange. Bound the recorded value to
|
||||
# the column's range; values outside become None (target_id is
|
||||
# nullable, so this is safe).
|
||||
_INT32_MAX = 2_147_483_647
|
||||
_INT32_MIN = -2_147_483_648
|
||||
|
||||
|
||||
def _queue_for(task) -> str:
|
||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||
Keep in sync if task_routes is reordered."""
|
||||
name = getattr(task, "name", "") or ""
|
||||
if name.startswith("backend.app.tasks.import_file."):
|
||||
return "import"
|
||||
if name.startswith("backend.app.tasks.ml."):
|
||||
return "ml"
|
||||
if name.startswith("backend.app.tasks.thumbnail."):
|
||||
return "thumbnail"
|
||||
if name.startswith("backend.app.tasks.download."):
|
||||
return "download"
|
||||
if name.startswith("backend.app.tasks.scan."):
|
||||
return "scan"
|
||||
if name.startswith((
|
||||
"backend.app.tasks.maintenance.",
|
||||
"backend.app.tasks.migration.",
|
||||
)):
|
||||
return "maintenance"
|
||||
return "default"
|
||||
|
||||
|
||||
def _target_id_from_args(args) -> int | None:
|
||||
"""Best-effort: if the first positional arg parses as int AND fits
|
||||
in the column's signed-32-bit range, record it as target_id
|
||||
(image_id, source_id, etc.). Never raises."""
|
||||
if not args:
|
||||
return None
|
||||
try:
|
||||
value = int(args[0])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if value < _INT32_MIN or value > _INT32_MAX:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _truncate(s, limit: int) -> str | None:
|
||||
if s is None:
|
||||
return None
|
||||
text = str(s)
|
||||
return text if len(text) <= limit else text[:limit]
|
||||
|
||||
|
||||
def _is_tracked(task_name: str | None) -> bool:
|
||||
return bool(task_name) and task_name not in _UNTRACKED_TASK_NAMES
|
||||
|
||||
|
||||
@task_prerun.connect
|
||||
def _on_prerun(sender=None, task_id=None, task=None, args=None,
|
||||
kwargs=None, **_):
|
||||
if not _is_tracked(getattr(task, "name", None)):
|
||||
return
|
||||
try:
|
||||
Session = sync_session_factory()
|
||||
with Session() as session:
|
||||
session.add(TaskRun(
|
||||
celery_task_id=task_id or "",
|
||||
queue=_queue_for(task),
|
||||
task_name=task.name,
|
||||
target_id=_target_id_from_args(args),
|
||||
started_at=datetime.now(UTC),
|
||||
status="running",
|
||||
args_summary=_truncate(repr(args), _MAX_ARGS_SUMMARY_LEN),
|
||||
worker_hostname=_truncate(
|
||||
getattr(sender, "hostname", None),
|
||||
_MAX_WORKER_HOSTNAME_LEN,
|
||||
),
|
||||
))
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001 — never break the worker
|
||||
log.exception("task_run prerun insert failed (task=%s)",
|
||||
getattr(task, "name", "?"))
|
||||
|
||||
|
||||
def _finalize(task_id: str, *, status: str,
|
||||
error_type: str | None = None,
|
||||
error_message: str | None = None,
|
||||
retry_count: int | None = None) -> None:
|
||||
"""Shared write path for postrun/failure/retry. Picks the most-
|
||||
recent task_run row for this celery_task_id that's still 'running'
|
||||
(retries reuse the same celery_task_id; each new attempt's prerun
|
||||
inserts a fresh row, so finalize targets the latest running row)."""
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
|
||||
Session = sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
with Session() as session:
|
||||
row = session.execute(
|
||||
select(TaskRun)
|
||||
.where(TaskRun.celery_task_id == task_id)
|
||||
.where(TaskRun.status == "running")
|
||||
.order_by(TaskRun.id.desc())
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return # no prerun row (untracked, insert failed, or already finalized)
|
||||
row.finished_at = now
|
||||
row.duration_ms = int(
|
||||
(now - row.started_at).total_seconds() * 1000
|
||||
)
|
||||
row.status = status
|
||||
if error_type is not None:
|
||||
row.error_type = _truncate(error_type, 128)
|
||||
if error_message is not None:
|
||||
row.error_message = _truncate(error_message, _MAX_ERROR_MESSAGE_LEN)
|
||||
if retry_count is not None:
|
||||
row.retry_count = retry_count
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("task_run finalize failed (task_id=%s)", task_id)
|
||||
|
||||
|
||||
@task_postrun.connect
|
||||
def _on_postrun(sender=None, task_id=None, task=None, args=None,
|
||||
kwargs=None, retval=None, state=None, **_):
|
||||
if not _is_tracked(getattr(task, "name", None)):
|
||||
return
|
||||
# state is one of SUCCESS/FAILURE/RETRY/etc. Only handle SUCCESS;
|
||||
# task_failure handles FAILURE explicitly (with the exception).
|
||||
if state != "SUCCESS":
|
||||
return
|
||||
_finalize(task_id, status="ok")
|
||||
|
||||
|
||||
@task_failure.connect
|
||||
def _on_failure(sender=None, task_id=None, exception=None,
|
||||
args=None, kwargs=None, einfo=None, **_):
|
||||
if not _is_tracked(getattr(sender, "name", None)):
|
||||
return
|
||||
status = ("timeout"
|
||||
if isinstance(exception, SoftTimeLimitExceeded)
|
||||
else "error")
|
||||
_finalize(
|
||||
task_id, status=status,
|
||||
error_type=type(exception).__name__ if exception else "Unknown",
|
||||
error_message=str(exception) if exception else None,
|
||||
)
|
||||
|
||||
|
||||
@task_retry.connect
|
||||
def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
||||
if not _is_tracked(getattr(sender, "name", None)):
|
||||
return
|
||||
task_id = getattr(request, "id", None)
|
||||
if not task_id:
|
||||
return
|
||||
# Mark current attempt's row as 'retry' (terminal for this row).
|
||||
# The next attempt's task_prerun inserts a fresh row.
|
||||
_finalize(
|
||||
task_id, status="retry",
|
||||
error_type=type(reason).__name__ if reason else "Retry",
|
||||
error_message=str(reason) if reason else None,
|
||||
retry_count=getattr(request, "retries", 0),
|
||||
)
|
||||
+44
-2
@@ -1,13 +1,18 @@
|
||||
"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback,
|
||||
and the on-disk image library + thumbnails from /images.
|
||||
the on-disk image library + thumbnails from /images, and the signed
|
||||
Firefox extension XPI from frontend/dist/extension/.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, abort, send_from_directory
|
||||
from quart import Blueprint, abort, send_file, send_from_directory
|
||||
|
||||
FRONTEND_DIST = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist"
|
||||
IMAGES_ROOT = Path("/images")
|
||||
XPI_DIR = FRONTEND_DIST / "extension"
|
||||
|
||||
_XPI_NAME_RE = re.compile(r"^fabledcurator-[\w.-]+\.xpi$")
|
||||
|
||||
frontend_bp = Blueprint("frontend", __name__)
|
||||
|
||||
@@ -31,6 +36,43 @@ async def serve_image(subpath: str):
|
||||
return await send_from_directory(IMAGES_ROOT, subpath)
|
||||
|
||||
|
||||
@frontend_bp.route("/extension/<filename>")
|
||||
async def serve_extension(filename: str):
|
||||
"""Serve the signed FC Firefox extension XPI.
|
||||
|
||||
Path whitelist: filename must match fabledcurator-*.xpi. The special
|
||||
name fabledcurator-latest.xpi serves the most-recently-modified XPI
|
||||
in the directory.
|
||||
|
||||
The application/x-xpinstall MIME tells Firefox to show its native
|
||||
install prompt instead of downloading the file as a blob.
|
||||
"""
|
||||
if not _XPI_NAME_RE.fullmatch(filename):
|
||||
abort(404)
|
||||
if not XPI_DIR.is_dir():
|
||||
abort(404)
|
||||
if filename == "fabledcurator-latest.xpi":
|
||||
xpis = sorted(XPI_DIR.glob("fabledcurator-*.xpi"), key=lambda p: p.stat().st_mtime)
|
||||
if not xpis:
|
||||
abort(404)
|
||||
latest = xpis[-1]
|
||||
return await send_file(
|
||||
latest, mimetype="application/x-xpinstall",
|
||||
attachment_filename=latest.name,
|
||||
)
|
||||
target = (XPI_DIR / filename).resolve()
|
||||
try:
|
||||
target.relative_to(XPI_DIR)
|
||||
except ValueError:
|
||||
abort(404)
|
||||
if not target.is_file():
|
||||
abort(404)
|
||||
return await send_file(
|
||||
target, mimetype="application/x-xpinstall",
|
||||
attachment_filename=filename,
|
||||
)
|
||||
|
||||
|
||||
@frontend_bp.route("/")
|
||||
@frontend_bp.route("/<path:subpath>")
|
||||
async def serve_spa(subpath: str = ""):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from .app_setting import AppSetting
|
||||
from .artist import Artist
|
||||
from .backup_run import BackupRun
|
||||
from .base import Base
|
||||
from .credential import Credential
|
||||
from .download_event import DownloadEvent
|
||||
@@ -21,11 +22,13 @@ from .tag_alias import TagAlias
|
||||
from .tag_allowlist import TagAllowlist
|
||||
from .tag_reference_embedding import TagReferenceEmbedding
|
||||
from .tag_suggestion_rejection import TagSuggestionRejection
|
||||
from .task_run import TaskRun
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"AppSetting",
|
||||
"Artist",
|
||||
"BackupRun",
|
||||
"Source",
|
||||
"Credential",
|
||||
"Post",
|
||||
@@ -46,4 +49,5 @@ __all__ = [
|
||||
"TagAllowlist",
|
||||
"TagReferenceEmbedding",
|
||||
"TagSuggestionRejection",
|
||||
"TaskRun",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""FC-3h: backup_run — operator-facing artifact record for a backup run.
|
||||
|
||||
One row per backup attempt (kind='db' or 'images'). Lifecycle
|
||||
tracking (started_at/finished_at/duration_ms/exception text) lives
|
||||
in task_run from FC-3i — this row records artifact metadata: file
|
||||
paths, sizes, tag (retention protection), and restore lineage via
|
||||
restored_from_id.
|
||||
|
||||
Status values (String, not Postgres ENUM — per
|
||||
feedback_check_existing_enums):
|
||||
pending — created but task hasn't started yet (rare; usually
|
||||
status starts as 'running' from the task body).
|
||||
running — backup task is in flight.
|
||||
ok — artifact successfully written.
|
||||
error — task raised; error column populated.
|
||||
restoring — this row represents a restore attempt (kind = restored
|
||||
kind); linked to source via restored_from_id.
|
||||
restored — restore completed successfully.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class BackupRun(Base):
|
||||
__tablename__ = "backup_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="pending", index=True,
|
||||
)
|
||||
tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True,
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True,
|
||||
)
|
||||
sql_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
tar_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
manifest: Mapped[dict] = mapped_column(
|
||||
JSON, nullable=False, default=dict, server_default="{}",
|
||||
)
|
||||
restored_from_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
@@ -49,3 +49,17 @@ class ImportSettings(Base):
|
||||
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=5
|
||||
)
|
||||
|
||||
# FC-3h backup knobs.
|
||||
backup_db_nightly_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False,
|
||||
)
|
||||
backup_db_nightly_hour_utc: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3,
|
||||
)
|
||||
backup_db_keep_last_n: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=14,
|
||||
)
|
||||
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""FC-3i: task_run — per-Celery-task lifecycle audit row.
|
||||
|
||||
One row inserted by the task_prerun signal at task start, updated by
|
||||
task_postrun / task_failure / task_retry. The shape supports the
|
||||
SystemActivity dashboard's three panes: per-queue queue+worker summary,
|
||||
recent failures (24h, grouped by error_type), and full paginated
|
||||
activity history.
|
||||
|
||||
Retention: ok rows pruned after 24h, error/timeout after 7d (see
|
||||
backend.app.tasks.maintenance.prune_task_runs).
|
||||
|
||||
Recovery: rows stuck in 'running' for >5 min flipped to 'error' by
|
||||
backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TaskRun(Base):
|
||||
__tablename__ = "task_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
celery_task_id: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, index=True,
|
||||
)
|
||||
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True,
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True,
|
||||
)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
)
|
||||
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
retry_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
worker_hostname: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
args_summary: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
@@ -0,0 +1,196 @@
|
||||
"""FC-3h: first-class backup/restore service for FC.
|
||||
|
||||
Two independent backup kinds:
|
||||
- 'db' — pg_dump only; fast; nightly via Beat (settings-gated)
|
||||
- 'images' — tar+zstd of /images; slow; manual trigger only
|
||||
|
||||
Files live under <images_root>/_backups/. Each backup writes:
|
||||
fc_<kind>_<ts>.{sql|tar.zst} — the artifact
|
||||
fc_<kind>_<ts>.json — manifest (kind/tag/triggered_by)
|
||||
|
||||
Service functions are sync (subprocess-bound). Celery tasks in
|
||||
backend.app.tasks.backup wrap each one with task_run-tracked
|
||||
lifecycle + soft/hard time limits + retention bookkeeping.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
|
||||
# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The
|
||||
# Celery soft limit signals the Python process; subprocess.Popen in a
|
||||
# blocking syscall ignores that signal. These bound the worst case.
|
||||
_DB_SUBPROCESS_TIMEOUT_S = 12 * 60 # 12 min (Celery soft is 10 min)
|
||||
_IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60 # 7 hr (Celery soft is 6 hr)
|
||||
|
||||
|
||||
def _libpq_url(sa_url: str) -> str:
|
||||
"""Strip SQLAlchemy +psycopg/+asyncpg driver suffix for pg_dump/psql."""
|
||||
for driver in (
|
||||
"postgresql+psycopg",
|
||||
"postgresql+asyncpg",
|
||||
"postgresql+psycopg2",
|
||||
):
|
||||
if sa_url.startswith(driver + "://"):
|
||||
return "postgresql://" + sa_url[len(driver) + 3:]
|
||||
return sa_url
|
||||
|
||||
|
||||
def _backups_dir(images_root: Path) -> Path:
|
||||
p = images_root / _BACKUPS_DIRNAME
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _now_ts() -> str:
|
||||
return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
def _file_size_or_none(path: Path) -> int | None:
|
||||
try:
|
||||
return path.stat().st_size
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _write_manifest(
|
||||
out_dir: Path, *, kind: str, ts: str,
|
||||
tag: str | None, triggered_by: str,
|
||||
artifact_path: Path,
|
||||
) -> Path:
|
||||
manifest = {
|
||||
"kind": kind,
|
||||
"backup_id": f"fc_{kind}_{ts}",
|
||||
"tag": tag,
|
||||
"triggered_by": triggered_by,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"artifact_path": str(artifact_path),
|
||||
}
|
||||
mf = out_dir / f"fc_{kind}_{ts}.json"
|
||||
mf.write_text(json.dumps(manifest, indent=2))
|
||||
return mf
|
||||
|
||||
|
||||
def backup_db(
|
||||
*, db_url: str, images_root: Path,
|
||||
tag: str | None = None, triggered_by: str = "manual",
|
||||
) -> dict:
|
||||
"""Run pg_dump; write .sql + manifest; return dict for the caller
|
||||
to persist into BackupRun. Raises on subprocess failure."""
|
||||
ts = _now_ts()
|
||||
out_dir = _backups_dir(images_root)
|
||||
sql_path = out_dir / f"fc_db_{ts}.sql"
|
||||
subprocess.run(
|
||||
[
|
||||
"pg_dump", "--no-owner", "--no-acl",
|
||||
"-f", str(sql_path), _libpq_url(db_url),
|
||||
],
|
||||
capture_output=True, check=True,
|
||||
timeout=_DB_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
manifest_path = _write_manifest(
|
||||
out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||
artifact_path=sql_path,
|
||||
)
|
||||
return {
|
||||
"kind": "db",
|
||||
"ts": ts,
|
||||
"sql_path": str(sql_path),
|
||||
"tar_path": None,
|
||||
"manifest_path": str(manifest_path),
|
||||
"size_bytes": _file_size_or_none(sql_path),
|
||||
}
|
||||
|
||||
|
||||
def backup_images(
|
||||
*, images_root: Path,
|
||||
tag: str | None = None, triggered_by: str = "manual",
|
||||
) -> dict:
|
||||
"""Run tar --zstd over images_root; write .tar.zst + manifest."""
|
||||
ts = _now_ts()
|
||||
out_dir = _backups_dir(images_root)
|
||||
tar_path = out_dir / f"fc_images_{ts}.tar.zst"
|
||||
subprocess.run(
|
||||
[
|
||||
"tar", "--zstd", "-cf", str(tar_path),
|
||||
"-C", str(images_root.parent), images_root.name,
|
||||
f"--exclude={images_root.name}/_backups",
|
||||
f"--exclude={images_root.name}/_quarantine",
|
||||
],
|
||||
capture_output=True, check=True,
|
||||
timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
manifest_path = _write_manifest(
|
||||
out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||
artifact_path=tar_path,
|
||||
)
|
||||
return {
|
||||
"kind": "images",
|
||||
"ts": ts,
|
||||
"sql_path": None,
|
||||
"tar_path": str(tar_path),
|
||||
"manifest_path": str(manifest_path),
|
||||
"size_bytes": _file_size_or_none(tar_path),
|
||||
}
|
||||
|
||||
|
||||
def restore_db(*, db_url: str, sql_path: Path) -> None:
|
||||
"""Wipe public schema, then load from .sql. Raises on subprocess
|
||||
failure; partial-restore state is the caller's concern."""
|
||||
libpq = _libpq_url(db_url)
|
||||
subprocess.run(
|
||||
[
|
||||
"psql", libpq, "-c",
|
||||
"DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
|
||||
],
|
||||
capture_output=True, check=True, timeout=120,
|
||||
)
|
||||
subprocess.run(
|
||||
["psql", libpq, "-f", str(sql_path)],
|
||||
capture_output=True, check=True,
|
||||
timeout=_DB_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
def restore_images(*, images_root: Path, tar_path: Path) -> None:
|
||||
"""Untar over images_root.parent. Additive — files NOT in the
|
||||
tarball are NOT removed. Caller wipes first if a clean restore
|
||||
is needed."""
|
||||
subprocess.run(
|
||||
[
|
||||
"tar", "--zstd", "-xf", str(tar_path),
|
||||
"-C", str(images_root.parent),
|
||||
],
|
||||
capture_output=True, check=True,
|
||||
timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
def unlink_artifact_files(
|
||||
*,
|
||||
sql_path: str | None,
|
||||
tar_path: str | None,
|
||||
manifest_path: str | None,
|
||||
) -> dict:
|
||||
"""Best-effort unlink of all on-disk files for a BackupRun row.
|
||||
Returns dict keyed by label with True/False per file. Missing
|
||||
files count as success (missing_ok semantics)."""
|
||||
deleted: dict = {}
|
||||
for label, p in (
|
||||
("sql", sql_path),
|
||||
("tar", tar_path),
|
||||
("manifest", manifest_path),
|
||||
):
|
||||
if not p:
|
||||
continue
|
||||
path = Path(p)
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
deleted[label] = True
|
||||
except OSError:
|
||||
deleted[label] = False
|
||||
return deleted
|
||||
@@ -0,0 +1,367 @@
|
||||
"""FC-3k: first-class admin destructive operations.
|
||||
|
||||
Projections are pure SELECTs used by both dry-run preview endpoints
|
||||
and Tier-B count prompts. Mutations (Task 2) are called from sync
|
||||
HTTP handlers (small ops) and from Celery tasks in
|
||||
backend.app.tasks.admin (long ops).
|
||||
|
||||
This module is the PERMANENT home of artist-cascade + image-unlink
|
||||
logic. The legacy copy at backend/app/services/migrators/cleanup.py
|
||||
stays in place until FC-3j; FC-3j will replace its body with thin
|
||||
re-exports from this module and then delete the wrapper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, ImageRecord, Tag
|
||||
from ..models.series_page import SeriesPage
|
||||
from ..models.tag import image_tag
|
||||
|
||||
|
||||
def project_artist_cascade(session: Session, *, slug: str) -> dict:
|
||||
"""Read-only projection of what delete_artist_cascade would touch.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"artist": {"id": int, "name": str, "slug": str},
|
||||
"projected": {
|
||||
"images": int,
|
||||
"sources": int,
|
||||
"thumbs": int, # images with a thumbnail_path set
|
||||
"import_tasks": int, # ImportTask rows referencing the artist's images
|
||||
"bytes_on_disk": int, # SUM(image_record.size_bytes) — column is NOT NULL
|
||||
},
|
||||
}
|
||||
Raises LookupError if slug not found. No mutations.
|
||||
"""
|
||||
from ..models.import_task import ImportTask
|
||||
from ..models.source import Source
|
||||
|
||||
artist = session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
).scalar_one_or_none()
|
||||
if artist is None:
|
||||
raise LookupError(f"artist slug not found: {slug!r}")
|
||||
|
||||
images_count = session.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.artist_id == artist.id)
|
||||
).scalar_one()
|
||||
sources_count = session.execute(
|
||||
select(func.count(Source.id))
|
||||
.where(Source.artist_id == artist.id)
|
||||
).scalar_one()
|
||||
thumbs_count = session.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.artist_id == artist.id)
|
||||
.where(ImageRecord.thumbnail_path.is_not(None))
|
||||
).scalar_one()
|
||||
import_tasks_count = session.execute(
|
||||
select(func.count(ImportTask.id))
|
||||
.where(
|
||||
ImportTask.result_image_id.in_(
|
||||
select(ImageRecord.id).where(ImageRecord.artist_id == artist.id)
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
bytes_on_disk = session.execute(
|
||||
select(func.coalesce(func.sum(ImageRecord.size_bytes), 0))
|
||||
.where(ImageRecord.artist_id == artist.id)
|
||||
).scalar_one()
|
||||
|
||||
return {
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"projected": {
|
||||
"images": images_count,
|
||||
"sources": sources_count,
|
||||
"thumbs": thumbs_count,
|
||||
"import_tasks": import_tasks_count,
|
||||
"bytes_on_disk": int(bytes_on_disk),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def project_bulk_image_delete(
|
||||
session: Session, *, image_ids: list[int],
|
||||
) -> dict:
|
||||
"""Read-only projection of what delete_images would touch.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"images_found": int,
|
||||
"thumbs_to_unlink": int,
|
||||
"bytes_on_disk": int,
|
||||
"missing_ids": list[int], # ids passed in that don't exist
|
||||
}
|
||||
No mutations.
|
||||
"""
|
||||
if not image_ids:
|
||||
return {
|
||||
"images_found": 0,
|
||||
"thumbs_to_unlink": 0,
|
||||
"bytes_on_disk": 0,
|
||||
"missing_ids": [],
|
||||
}
|
||||
|
||||
rows = session.execute(
|
||||
select(
|
||||
ImageRecord.id,
|
||||
ImageRecord.thumbnail_path,
|
||||
ImageRecord.size_bytes,
|
||||
).where(ImageRecord.id.in_(image_ids))
|
||||
).all()
|
||||
found_ids = {r.id for r in rows}
|
||||
missing = sorted(set(image_ids) - found_ids)
|
||||
return {
|
||||
"images_found": len(rows),
|
||||
"thumbs_to_unlink": sum(1 for r in rows if r.thumbnail_path),
|
||||
"bytes_on_disk": sum(r.size_bytes for r in rows),
|
||||
"missing_ids": missing,
|
||||
}
|
||||
|
||||
|
||||
def count_tag_associations(session: Session, *, tag_id: int) -> int:
|
||||
"""COUNT(*) FROM image_tag WHERE tag_id=?. For Tier-B prompt."""
|
||||
return session.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
).scalar_one()
|
||||
|
||||
|
||||
def find_unused_tags(
|
||||
session: Session, *, limit: int | None = None,
|
||||
) -> list[Tag]:
|
||||
"""Tags with no image_tag rows AND no series_page rows.
|
||||
|
||||
Sorted by name. Used by both dry-run preview and the live prune.
|
||||
A tag is "unused" iff it has zero rows in image_tag AND zero rows
|
||||
in series_page (so we don't accidentally prune a series tag that
|
||||
happens to have no images yet).
|
||||
"""
|
||||
used_via_image_tag = select(image_tag.c.tag_id).distinct()
|
||||
used_via_series = select(SeriesPage.series_tag_id).where(
|
||||
SeriesPage.series_tag_id.is_not(None)
|
||||
).distinct()
|
||||
stmt = (
|
||||
select(Tag)
|
||||
.where(Tag.id.not_in(used_via_image_tag))
|
||||
.where(Tag.id.not_in(used_via_series))
|
||||
.order_by(Tag.name)
|
||||
)
|
||||
if limit is not None:
|
||||
stmt = stmt.limit(limit)
|
||||
return list(session.execute(stmt).scalars().all())
|
||||
|
||||
|
||||
def unlink_image_files(
|
||||
image: ImageRecord, images_root: Path,
|
||||
) -> dict:
|
||||
"""Best-effort unlink of all on-disk files for an ImageRecord.
|
||||
|
||||
Targets: image.path (original), image.thumbnail_path (cached
|
||||
thumbnail), and the computed thumbs path at
|
||||
/images/thumbs/<sha256[:3]>/<sha256>.(jpg|png|webp) (tries all
|
||||
three extensions; missing extension is silently OK).
|
||||
|
||||
Returns {"original": bool, "thumbnail": bool}. Missing files
|
||||
count as success (missing_ok semantics). OSErrors are swallowed
|
||||
and reported as False so the calling DB delete still proceeds.
|
||||
"""
|
||||
out = {"original": False, "thumbnail": False}
|
||||
if image.path:
|
||||
try:
|
||||
Path(image.path).unlink(missing_ok=True)
|
||||
out["original"] = True
|
||||
except OSError:
|
||||
out["original"] = False
|
||||
# Custom thumbnail_path (when set) — try it first.
|
||||
if image.thumbnail_path:
|
||||
try:
|
||||
Path(image.thumbnail_path).unlink(missing_ok=True)
|
||||
out["thumbnail"] = True
|
||||
except OSError:
|
||||
out["thumbnail"] = False
|
||||
# Convention thumbs dir — try all extensions; missing OK.
|
||||
if image.sha256:
|
||||
bucket = image.sha256[:3]
|
||||
for ext in ("jpg", "png", "webp"):
|
||||
try:
|
||||
(images_root / "thumbs" / bucket / f"{image.sha256}.{ext}").unlink(
|
||||
missing_ok=True,
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def delete_artist_cascade(
|
||||
session: Session, *, artist_id: int, images_root: Path,
|
||||
) -> dict:
|
||||
"""Batched delete of an artist's images + the artist row.
|
||||
|
||||
Mirrors the cleanup_artist_async pattern: 500-row batches,
|
||||
commit between batches so partial progress survives a worker
|
||||
kill. Idempotent on missing artist (returns zeroed counts).
|
||||
Postgres cascades handle image_tag / image_provenance /
|
||||
series_page / tag_suggestion_rejection from ImageRecord delete,
|
||||
and source / post / download_event / etc. from Artist delete
|
||||
(via Artist.sources cascade="all, delete-orphan").
|
||||
"""
|
||||
artist = session.get(Artist, artist_id)
|
||||
if artist is None:
|
||||
return {
|
||||
"artist": None,
|
||||
"summary": {
|
||||
"images_deleted": 0,
|
||||
"files_deleted": 0,
|
||||
"thumbs_deleted": 0,
|
||||
"import_tasks_nulled": 0,
|
||||
"files_failed": 0,
|
||||
},
|
||||
}
|
||||
artist_info = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
|
||||
images_deleted = 0
|
||||
files_deleted = 0
|
||||
thumbs_deleted = 0
|
||||
files_failed = 0
|
||||
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord)
|
||||
.where(ImageRecord.artist_id == artist.id)
|
||||
.limit(500)
|
||||
).scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
for img in rows:
|
||||
unlinked = unlink_image_files(img, images_root)
|
||||
if unlinked["original"]:
|
||||
files_deleted += 1
|
||||
else:
|
||||
files_failed += 1
|
||||
if unlinked["thumbnail"]:
|
||||
thumbs_deleted += 1
|
||||
session.delete(img)
|
||||
images_deleted += 1
|
||||
session.commit()
|
||||
|
||||
# ImportTask.result_image_id FK is SET NULL on image delete (Postgres
|
||||
# handles this in the cascade above). We don't separately count those
|
||||
# in FC-3k — the legacy cleanup_artist_async did it via
|
||||
# source_path_prefix matching that's out of scope here.
|
||||
import_tasks_nulled = 0
|
||||
|
||||
session.delete(artist)
|
||||
session.commit()
|
||||
|
||||
return {
|
||||
"artist": artist_info,
|
||||
"summary": {
|
||||
"images_deleted": images_deleted,
|
||||
"files_deleted": files_deleted,
|
||||
"thumbs_deleted": thumbs_deleted,
|
||||
"import_tasks_nulled": import_tasks_nulled,
|
||||
"files_failed": files_failed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def delete_images(
|
||||
session: Session, *, image_ids: list[int], images_root: Path,
|
||||
) -> dict:
|
||||
"""Delete a list of images in 500-row batches with commit between.
|
||||
|
||||
Postgres CASCADE on image_tag / image_provenance / series_page /
|
||||
tag_suggestion_rejection / post_attachment(FK SET NULL) handles
|
||||
the DB side; this function handles file unlinks first then row
|
||||
deletes. Idempotent on missing IDs (returned as missing_ids;
|
||||
no error). On partial OSError, the row is still deleted and
|
||||
files_failed is incremented.
|
||||
"""
|
||||
if not image_ids:
|
||||
return {
|
||||
"images_deleted": 0,
|
||||
"files_deleted": 0,
|
||||
"thumbs_deleted": 0,
|
||||
"files_failed": 0,
|
||||
"missing_ids": [],
|
||||
}
|
||||
|
||||
seen_ids: set[int] = set()
|
||||
images_deleted = 0
|
||||
files_deleted = 0
|
||||
thumbs_deleted = 0
|
||||
files_failed = 0
|
||||
|
||||
pending = list(image_ids)
|
||||
while pending:
|
||||
batch_ids = pending[:500]
|
||||
pending = pending[500:]
|
||||
rows = session.execute(
|
||||
select(ImageRecord).where(ImageRecord.id.in_(batch_ids))
|
||||
).scalars().all()
|
||||
for img in rows:
|
||||
seen_ids.add(img.id)
|
||||
unlinked = unlink_image_files(img, images_root)
|
||||
if unlinked["original"]:
|
||||
files_deleted += 1
|
||||
else:
|
||||
files_failed += 1
|
||||
if unlinked["thumbnail"]:
|
||||
thumbs_deleted += 1
|
||||
session.delete(img)
|
||||
images_deleted += 1
|
||||
session.commit()
|
||||
|
||||
missing = sorted(set(image_ids) - seen_ids)
|
||||
return {
|
||||
"images_deleted": images_deleted,
|
||||
"files_deleted": files_deleted,
|
||||
"thumbs_deleted": thumbs_deleted,
|
||||
"files_failed": files_failed,
|
||||
"missing_ids": missing,
|
||||
}
|
||||
|
||||
|
||||
def delete_tag(session: Session, *, tag_id: int) -> dict:
|
||||
"""Simple DELETE FROM tag WHERE id=?.
|
||||
|
||||
Postgres cascades the rest (image_tag, tag_alias, tag_allowlist,
|
||||
tag_reference_embedding, tag_suggestion_rejection, series_page).
|
||||
Returns counts BEFORE delete so the caller can surface them.
|
||||
Raises LookupError if tag_id not found.
|
||||
"""
|
||||
tag = session.get(Tag, tag_id)
|
||||
if tag is None:
|
||||
raise LookupError(f"tag id not found: {tag_id}")
|
||||
associations_count = count_tag_associations(session, tag_id=tag_id)
|
||||
info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
|
||||
session.delete(tag)
|
||||
session.commit()
|
||||
return {"deleted": info, "associations_removed": associations_count}
|
||||
|
||||
|
||||
def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||
"""Find tags with zero references and (unless dry_run) delete them.
|
||||
|
||||
Returns:
|
||||
dry_run=True: {"count": N, "sample_names": [first 50]}
|
||||
dry_run=False: {"deleted": N, "sample_names": [first 50]}
|
||||
"""
|
||||
unused = find_unused_tags(session)
|
||||
sample = [t.name for t in unused[:50]]
|
||||
if dry_run:
|
||||
return {"count": len(unused), "sample_names": sample}
|
||||
ids = [t.id for t in unused]
|
||||
if ids:
|
||||
session.execute(
|
||||
Tag.__table__.delete().where(Tag.id.in_(ids))
|
||||
)
|
||||
session.commit()
|
||||
return {"deleted": len(ids), "sample_names": sample}
|
||||
@@ -0,0 +1,127 @@
|
||||
"""FC-3g: backend support for the Firefox extension.
|
||||
|
||||
`ExtensionService.quick_add_source(url)` derives platform + artist
|
||||
slug from a URL using regex patterns mirrored from
|
||||
extension/lib/platforms.js, then find-or-creates Artist + Source rows
|
||||
and returns a JSON-shaped dict for the API layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, Source
|
||||
from ..utils.slug import slugify
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
|
||||
|
||||
class InvalidUrlError(Exception):
|
||||
"""URL was empty or missing a scheme."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
_PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("patreon", re.compile(
|
||||
r"^https?://(?:www\.)?patreon\.com/"
|
||||
r"(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c/)"
|
||||
r"(?P<slug>[^/?#]+)/?$",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
("subscribestar", re.compile(
|
||||
r"^https?://(?:www\.)?subscribestar\.(?:com|adult)/"
|
||||
r"(?!feed$|messages$|library$)"
|
||||
r"(?P<slug>[^/?#]+)/?$",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
("hentaifoundry", re.compile(
|
||||
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
("deviantart", re.compile(
|
||||
r"^https?://(?:www\.)?deviantart\.com/"
|
||||
r"(?!home$|watch\b|tag\b|browse\b)"
|
||||
r"(?P<slug>[^/?#]+)/?$",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
("pixiv", re.compile(
|
||||
r"^https?://(?:www\.)?pixiv\.net/(?:en/)?users/(?P<slug>\d+)",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
]
|
||||
|
||||
|
||||
class ExtensionService:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self.session = session
|
||||
|
||||
async def quick_add_source(self, url: str) -> dict:
|
||||
platform, raw_slug = self._derive(url)
|
||||
artist, created_artist = await self._find_or_create_artist(raw_slug)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
return {
|
||||
"source": {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
},
|
||||
"artist": {
|
||||
"id": artist.id,
|
||||
"name": artist.name,
|
||||
"slug": artist.slug,
|
||||
},
|
||||
"created_source": created_source,
|
||||
"created_artist": created_artist,
|
||||
}
|
||||
|
||||
def _derive(self, url: str) -> tuple[str, str]:
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
raise InvalidUrlError("url is empty")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise InvalidUrlError(f"url must include http:// or https:// scheme: {url!r}")
|
||||
for platform, pattern in _PLATFORM_PATTERNS:
|
||||
m = pattern.match(url)
|
||||
if m:
|
||||
return platform, m.group("slug")
|
||||
raise UnknownPlatformError(f"no platform pattern matched {url!r}")
|
||||
|
||||
async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]:
|
||||
slug = slugify(raw_name)
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
||||
self.session.add(artist)
|
||||
await self.session.flush()
|
||||
return artist, True
|
||||
|
||||
async def _find_or_create_source(
|
||||
self, *, artist_id: int, platform: str, url: str,
|
||||
) -> tuple[Source, bool]:
|
||||
existing = (await self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
src = Source(artist_id=artist_id, platform=platform, url=url, enabled=True)
|
||||
self.session.add(src)
|
||||
await self.session.flush()
|
||||
await self.session.commit()
|
||||
return src, True
|
||||
@@ -237,6 +237,15 @@ class GalleryService:
|
||||
)
|
||||
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
||||
neighbors = await self._neighbors(record)
|
||||
# Direct artist FK — used by the modal's ProvenancePanel as a
|
||||
# fallback when ImageProvenance is empty (i.e., filesystem-
|
||||
# imported images without a post-track provenance row). The
|
||||
# source of truth for richer post-level data is still
|
||||
# ImageProvenance/Post; this is just the "we at least know who
|
||||
# made it" line.
|
||||
artist = None
|
||||
if record.artist_id is not None:
|
||||
artist = await self.session.get(Artist, record.artist_id)
|
||||
return {
|
||||
"id": record.id,
|
||||
"path": record.path,
|
||||
@@ -249,6 +258,10 @@ class GalleryService:
|
||||
"created_at": record.created_at.isoformat(),
|
||||
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
|
||||
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
||||
"artist": (
|
||||
{"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
if artist is not None else None
|
||||
),
|
||||
"tags": [
|
||||
{
|
||||
"id": t.id,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
"""FC-5 migration tooling.
|
||||
|
||||
One module per concern (backup/rollback/gs/ir/overlap/ml_queue/verify).
|
||||
One module per concern (gs/ir/overlap/ml_queue/verify/cleanup).
|
||||
Each migrator returns a counts dict; the run_migration task wires
|
||||
that dict into MigrationRun.counts so the UI polling shows progress.
|
||||
|
||||
backup + rollback were retired in FC-3h (2026-05-24); first-class
|
||||
backup lives at backend/app/services/backup_service.py and exposes
|
||||
its own /api/system/backup/* surface.
|
||||
"""
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
"""pg_dump + tar.zst-based backup, restorable via pair of subprocess calls.
|
||||
|
||||
Backups live under <images_root>/_backups/. Each backup is two files
|
||||
(SQL + tarball) plus a manifest JSON. Tagged backups (e.g. tag='pre_migration')
|
||||
are how rollback.py finds the most recent restorable snapshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
|
||||
|
||||
def _libpq_url(sa_url: str) -> str:
|
||||
"""Strip SQLAlchemy driver suffix so pg_dump/psql accept the URL.
|
||||
|
||||
SQLAlchemy uses URLs like `postgresql+psycopg://...` or
|
||||
`postgresql+asyncpg://...`. libpq tools (pg_dump, psql) only know
|
||||
the plain `postgresql://` scheme.
|
||||
"""
|
||||
for driver in ("postgresql+psycopg", "postgresql+asyncpg", "postgresql+psycopg2"):
|
||||
if sa_url.startswith(driver + "://"):
|
||||
return "postgresql://" + sa_url[len(driver) + 3:]
|
||||
return sa_url
|
||||
|
||||
|
||||
def _backups_dir(images_root: Path | None = None) -> Path:
|
||||
# Overridable for tests via monkeypatch.
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
p = root / _BACKUPS_DIRNAME
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _run_subprocess(cmd: list[str], **kwargs: Any):
|
||||
# Overridable for tests via monkeypatch.
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
**{k: v for k, v in kwargs.items() if not k.startswith("_")},
|
||||
)
|
||||
|
||||
|
||||
def create_backup(
|
||||
*, db_url: str, images_root: Path, tag: str = "manual",
|
||||
) -> dict:
|
||||
"""Create a backup: pg_dump SQL + tar.zst of images.
|
||||
|
||||
Returns a manifest dict. Writes <ts>.sql, <ts>.tar.zst, <ts>.json into
|
||||
<images_root>/_backups/.
|
||||
"""
|
||||
ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
out_dir = _backups_dir(images_root)
|
||||
sql_path = out_dir / f"fc_{ts}.sql"
|
||||
tar_path = out_dir / f"fc_{ts}.tar.zst"
|
||||
manifest_path = out_dir / f"fc_{ts}.json"
|
||||
|
||||
_run_subprocess(
|
||||
["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), _libpq_url(db_url)],
|
||||
_test_ts=ts,
|
||||
)
|
||||
_run_subprocess(
|
||||
[
|
||||
"tar", "--zstd", "-cf", str(tar_path),
|
||||
"-C", str(images_root.parent), images_root.name,
|
||||
f"--exclude={images_root.name}/_backups",
|
||||
f"--exclude={images_root.name}/_quarantine",
|
||||
],
|
||||
_test_ts=ts,
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"backup_id": ts,
|
||||
"tag": tag,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"sql_path": str(sql_path),
|
||||
"tar_path": str(tar_path),
|
||||
}
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2))
|
||||
return manifest
|
||||
|
||||
|
||||
def list_backups(images_root: Path) -> list[dict]:
|
||||
out_dir = _backups_dir(images_root)
|
||||
items = []
|
||||
for mf in sorted(out_dir.glob("fc_*.json"), reverse=True):
|
||||
try:
|
||||
items.append(json.loads(mf.read_text()))
|
||||
except Exception:
|
||||
continue
|
||||
return items
|
||||
|
||||
|
||||
def find_latest_backup(images_root: Path, *, tag: str) -> dict | None:
|
||||
for mf in list_backups(images_root):
|
||||
if mf.get("tag") == tag:
|
||||
return mf
|
||||
return None
|
||||
|
||||
|
||||
def restore_backup(
|
||||
*, manifest: dict, db_url: str, images_root: Path,
|
||||
) -> dict:
|
||||
"""Restore from a backup manifest.
|
||||
|
||||
1. Replay the .sql via psql.
|
||||
2. Wipe /images/ contents (except _backups/, which holds the file we're using).
|
||||
3. Untar the .tar.zst into /images/.
|
||||
"""
|
||||
sql_path = Path(manifest["sql_path"])
|
||||
tar_path = Path(manifest["tar_path"])
|
||||
|
||||
_run_subprocess(
|
||||
["psql", "-d", _libpq_url(db_url), "-f", str(sql_path)],
|
||||
)
|
||||
|
||||
# Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup
|
||||
# we're restoring from!).
|
||||
for entry in images_root.iterdir():
|
||||
if entry.name == _BACKUPS_DIRNAME:
|
||||
continue
|
||||
if entry.is_dir():
|
||||
shutil.rmtree(entry)
|
||||
else:
|
||||
entry.unlink()
|
||||
|
||||
_run_subprocess(
|
||||
["tar", "--zstd", "-xf", str(tar_path), "-C", str(images_root.parent)],
|
||||
)
|
||||
|
||||
return {"restored_from": manifest["backup_id"]}
|
||||
@@ -70,7 +70,7 @@ async def migrate_async(
|
||||
"""
|
||||
if data.get("source_app") != "imagerepo":
|
||||
raise ValueError("export source_app must be 'imagerepo'")
|
||||
if data.get("schema_version") != 1:
|
||||
if data.get("schema_version") not in (1, 2):
|
||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
||||
|
||||
counts = _zero_counts()
|
||||
@@ -126,15 +126,20 @@ async def migrate_async(
|
||||
await db.commit()
|
||||
|
||||
# Phase 2: write the per-image manifest for tag_apply.py to consume later.
|
||||
# schema_version 2 (added 2026-05-24) carries `image_posts` for
|
||||
# Post + Source + ImageProvenance restore; schema 1 manifests
|
||||
# without it stay valid (tag_apply treats the missing field as []).
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"schema_version": data.get("schema_version", 1),
|
||||
"image_artist_assignments": data.get("image_artist_assignments", []),
|
||||
"image_tag_associations": data.get("image_tag_associations", []),
|
||||
"series_pages": data.get("series_pages", []),
|
||||
"image_posts": data.get("image_posts", []),
|
||||
}
|
||||
counts["rows_processed"] += len(manifest["image_artist_assignments"])
|
||||
counts["rows_processed"] += len(manifest["image_tag_associations"])
|
||||
counts["rows_processed"] += len(manifest["series_pages"])
|
||||
counts["rows_processed"] += len(manifest["image_posts"])
|
||||
if not dry_run:
|
||||
manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
|
||||
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Restore from the most recent 'pre_migration'-tagged backup."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from . import backup as backup_mod
|
||||
|
||||
|
||||
class NoBackupFoundError(Exception):
|
||||
"""Raised when rollback() is called with no pre_migration backup on disk."""
|
||||
|
||||
|
||||
def rollback_to_pre_migration(*, db_url: str, images_root: Path) -> dict:
|
||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
||||
if manifest is None:
|
||||
raise NoBackupFoundError(
|
||||
"no pre_migration-tagged backup found under <images_root>/_backups/"
|
||||
)
|
||||
return backup_mod.restore_backup(
|
||||
manifest=manifest, db_url=db_url, images_root=images_root,
|
||||
)
|
||||
@@ -7,6 +7,7 @@ FC's filesystem scan over the mounted IR images dir).
|
||||
- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug).
|
||||
- image_tag_associations → image_tag insert (idempotent).
|
||||
- series_pages → series_page insert (idempotent on image_id unique).
|
||||
- image_posts (schema v2) → Source + Post + ImageProvenance restore.
|
||||
|
||||
Unmatched sha256s are logged into the result's `unmatched` list so the
|
||||
Celery task can drop them into MigrationRun.metadata for the operator
|
||||
@@ -15,15 +16,118 @@ to inspect.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag
|
||||
from ...models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
SeriesPage,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
image_tag,
|
||||
)
|
||||
from ...utils.slug import slugify
|
||||
from .ir_ingest import manifest_path
|
||||
|
||||
# Per-platform artist-profile URL — used as Source.url when restoring
|
||||
# IR PostMetadata into FC. Keep this table in sync with
|
||||
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS and
|
||||
# extension/lib/platforms.js.
|
||||
_PLATFORM_PROFILE_URL = {
|
||||
"patreon": "https://www.patreon.com/{slug}",
|
||||
"subscribestar": "https://www.subscribestar.com/{slug}",
|
||||
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
|
||||
}
|
||||
|
||||
|
||||
def _profile_url(platform: str, artist_slug: str) -> str | None:
|
||||
fmt = _PLATFORM_PROFILE_URL.get(platform)
|
||||
return fmt.format(slug=artist_slug) if fmt else None
|
||||
|
||||
|
||||
async def _find_or_create_source(
|
||||
db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool,
|
||||
) -> int | None:
|
||||
existing = (await db.execute(
|
||||
select(Source.id).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
if dry_run:
|
||||
return None
|
||||
s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False)
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
return s.id
|
||||
|
||||
|
||||
async def _find_or_create_post(
|
||||
db: AsyncSession, *,
|
||||
source_id: int, external_post_id: str,
|
||||
title: str | None, description: str | None, post_url: str | None,
|
||||
post_date_iso: str | None, attachment_count: int, dry_run: bool,
|
||||
) -> int | None:
|
||||
existing = (await db.execute(
|
||||
select(Post.id).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
if dry_run:
|
||||
return None
|
||||
post_date = None
|
||||
if post_date_iso:
|
||||
post_date = datetime.fromisoformat(post_date_iso)
|
||||
p = Post(
|
||||
source_id=source_id,
|
||||
external_post_id=external_post_id,
|
||||
post_title=title,
|
||||
description=description,
|
||||
post_url=post_url,
|
||||
post_date=post_date,
|
||||
attachment_count=attachment_count,
|
||||
raw_metadata={"migrated_from": "imagerepo"},
|
||||
)
|
||||
db.add(p)
|
||||
await db.flush()
|
||||
return p.id
|
||||
|
||||
|
||||
async def _ensure_provenance(
|
||||
db: AsyncSession, *,
|
||||
image_id: int, post_id: int, source_id: int, dry_run: bool,
|
||||
) -> bool:
|
||||
"""Returns True if a new ImageProvenance row was inserted."""
|
||||
existing = (await db.execute(
|
||||
select(ImageProvenance.id).where(
|
||||
ImageProvenance.image_record_id == image_id,
|
||||
ImageProvenance.post_id == post_id,
|
||||
ImageProvenance.source_id == source_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return False
|
||||
if dry_run:
|
||||
return True
|
||||
db.add(ImageProvenance(
|
||||
image_record_id=image_id, post_id=post_id, source_id=source_id,
|
||||
))
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
@@ -167,6 +271,67 @@ async def apply_async(
|
||||
))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
# 4. Image posts (schema v2) → Source + Post + ImageProvenance.
|
||||
# Restores IR PostMetadata as FC's downloader-track provenance,
|
||||
# so the modal's ProvenancePanel surfaces title/description/
|
||||
# source URL/publish date the same way it does for live
|
||||
# gallery-dl downloads.
|
||||
for entry in manifest.get("image_posts", []):
|
||||
counts["rows_processed"] += 1
|
||||
platform = entry.get("platform")
|
||||
artist_name = entry.get("artist")
|
||||
if not platform or not artist_name:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
|
||||
aid = await _ensure_artist_id(db, artist_name, dry_run)
|
||||
if aid is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
|
||||
url = _profile_url(platform, slugify(artist_name))
|
||||
if url is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
|
||||
source_id = await _find_or_create_source(
|
||||
db, artist_id=aid, platform=platform, url=url, dry_run=dry_run,
|
||||
)
|
||||
if source_id is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
|
||||
post_id = await _find_or_create_post(
|
||||
db, source_id=source_id,
|
||||
external_post_id=entry.get("post_id") or "",
|
||||
title=entry.get("title"),
|
||||
description=entry.get("description"),
|
||||
post_url=entry.get("source_url"),
|
||||
post_date_iso=entry.get("published_at"),
|
||||
attachment_count=entry.get("attachment_count") or 0,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if post_id is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
|
||||
for sha in entry.get("image_sha256s", []):
|
||||
img_id = await _sha_to_image_id(db, sha)
|
||||
if img_id is None:
|
||||
unmatched.append({
|
||||
"kind": "post", "sha256": sha,
|
||||
"post_id": entry.get("post_id"),
|
||||
})
|
||||
continue
|
||||
inserted = await _ensure_provenance(
|
||||
db, image_id=img_id, post_id=post_id,
|
||||
source_id=source_id, dry_run=dry_run,
|
||||
)
|
||||
if inserted:
|
||||
counts["rows_inserted"] += 1
|
||||
else:
|
||||
counts["rows_skipped"] += 1
|
||||
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
return {"counts": counts, "unmatched": unmatched}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""FC-3k: admin destructive Celery tasks.
|
||||
|
||||
Two long-running ops on the maintenance queue. task_run lifecycle is
|
||||
captured automatically by FC-3i signals — these tasks just return
|
||||
their summary dict so it lands in task_run.metadata (via Celery's
|
||||
result backend) for the dashboard to surface.
|
||||
|
||||
Soft/hard time limits inherit the FC-3i recovery sweep: a runaway
|
||||
task gets killed and flipped to status='timeout' by
|
||||
recover_stalled_task_runs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..services import cleanup_service
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.delete_artist_cascade_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def delete_artist_cascade_task(self, *, artist_id: int) -> dict:
|
||||
"""Wraps cleanup_service.delete_artist_cascade. Returns the
|
||||
service's summary dict for FC-3i task_run.metadata capture."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return cleanup_service.delete_artist_cascade(
|
||||
session, artist_id=artist_id, images_root=IMAGES_ROOT,
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.bulk_delete_images_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=900, time_limit=1200, # 15 min / 20 min
|
||||
)
|
||||
def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
|
||||
"""Wraps cleanup_service.delete_images."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return cleanup_service.delete_images(
|
||||
session, image_ids=image_ids, images_root=IMAGES_ROOT,
|
||||
)
|
||||
@@ -0,0 +1,300 @@
|
||||
"""FC-3h: backup/restore Celery tasks.
|
||||
|
||||
All tasks live on the maintenance queue (per celery_app.task_routes).
|
||||
task_run lifecycle tracking is automatic via FC-3i signals — these
|
||||
tasks just record the operator-facing artifact metadata into
|
||||
BackupRun.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import BackupRun, ImportSettings
|
||||
from ..services import backup_service
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
|
||||
"""Flip a BackupRun row from running/restoring to error with a
|
||||
truncated error message and finished_at. Caller already holds the
|
||||
session open."""
|
||||
row.status = "error"
|
||||
row.error = f"{type(exc).__name__}: {exc}"[:2000]
|
||||
row.finished_at = datetime.now(UTC)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.backup_db_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=10, retry_backoff_max=120, max_retries=2,
|
||||
soft_time_limit=600, time_limit=720,
|
||||
)
|
||||
def backup_db_task(self, *, tag: str | None = None,
|
||||
triggered_by: str = "manual") -> dict:
|
||||
"""Create one DB backup. Returns {'backup_run_id': N}."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cfg = get_config()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
row = BackupRun(
|
||||
kind="db", status="running", tag=tag,
|
||||
triggered_by=triggered_by, started_at=now, manifest={},
|
||||
)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
run_id = row.id
|
||||
|
||||
try:
|
||||
result = backup_service.backup_db(
|
||||
db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
|
||||
tag=tag, triggered_by=triggered_by,
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
row.sql_path = result["sql_path"]
|
||||
row.size_bytes = result["size_bytes"]
|
||||
row.manifest = {
|
||||
"manifest_path": result["manifest_path"],
|
||||
"ts": result["ts"],
|
||||
}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.backup_images_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=30, retry_backoff_max=300, max_retries=1,
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def backup_images_task(self, *, tag: str | None = None,
|
||||
triggered_by: str = "manual") -> dict:
|
||||
"""Create one images backup. Same shape as backup_db_task; uses
|
||||
tar_path instead of sql_path."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
row = BackupRun(
|
||||
kind="images", status="running", tag=tag,
|
||||
triggered_by=triggered_by, started_at=now, manifest={},
|
||||
)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
run_id = row.id
|
||||
|
||||
try:
|
||||
result = backup_service.backup_images(
|
||||
images_root=IMAGES_ROOT,
|
||||
tag=tag, triggered_by=triggered_by,
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
row.tar_path = result["tar_path"]
|
||||
row.size_bytes = result["size_bytes"]
|
||||
row.manifest = {
|
||||
"manifest_path": result["manifest_path"],
|
||||
"ts": result["ts"],
|
||||
}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.restore_db_task",
|
||||
bind=True,
|
||||
max_retries=0, # NEVER auto-retry a half-applied restore.
|
||||
soft_time_limit=1200, time_limit=1800,
|
||||
)
|
||||
def restore_db_task(self, *, source_backup_run_id: int) -> dict:
|
||||
"""Restore from a previous DB backup. Inserts a NEW BackupRun row
|
||||
(kind='db', status='restoring') linked to the source via
|
||||
restored_from_id; flips to 'restored' on success or 'error' on
|
||||
failure. Operator sees the restore as a row in the dashboard."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cfg = get_config()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
src = session.get(BackupRun, source_backup_run_id)
|
||||
if src is None or src.kind != "db" or not src.sql_path:
|
||||
raise ValueError(
|
||||
f"BackupRun id={source_backup_run_id} is not a valid DB backup"
|
||||
)
|
||||
marker = BackupRun(
|
||||
kind="db", status="restoring",
|
||||
triggered_by="restore", started_at=now,
|
||||
restored_from_id=src.id,
|
||||
manifest={"source_sql_path": src.sql_path},
|
||||
)
|
||||
session.add(marker)
|
||||
session.commit()
|
||||
session.refresh(marker)
|
||||
marker_id = marker.id
|
||||
sql_path = src.sql_path
|
||||
|
||||
try:
|
||||
backup_service.restore_db(
|
||||
db_url=cfg.database_url_sync, sql_path=Path(sql_path),
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
row.status = "restored"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return {"backup_run_id": marker_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.restore_images_task",
|
||||
bind=True, max_retries=0,
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def restore_images_task(self, *, source_backup_run_id: int) -> dict:
|
||||
"""Mirrors restore_db_task; uses backup_service.restore_images."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
src = session.get(BackupRun, source_backup_run_id)
|
||||
if src is None or src.kind != "images" or not src.tar_path:
|
||||
raise ValueError(
|
||||
f"BackupRun id={source_backup_run_id} is not a valid images backup"
|
||||
)
|
||||
marker = BackupRun(
|
||||
kind="images", status="restoring",
|
||||
triggered_by="restore", started_at=now,
|
||||
restored_from_id=src.id,
|
||||
manifest={"source_tar_path": src.tar_path},
|
||||
)
|
||||
session.add(marker)
|
||||
session.commit()
|
||||
session.refresh(marker)
|
||||
marker_id = marker.id
|
||||
tar_path = src.tar_path
|
||||
|
||||
try:
|
||||
backup_service.restore_images(
|
||||
images_root=IMAGES_ROOT, tar_path=Path(tar_path),
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
row.status = "restored"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return {"backup_run_id": marker_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.prune_backups",
|
||||
soft_time_limit=300, time_limit=600,
|
||||
)
|
||||
def prune_backups() -> dict:
|
||||
"""Daily Beat. Per-kind retention from ImportSettings.
|
||||
|
||||
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
||||
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||
Status='running' / 'restoring' rows are never pruned (recovery
|
||||
sweep from FC-3i handles those via task_run).
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||
with SessionLocal() as session:
|
||||
s = session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
for kind, keep in (
|
||||
("db", s.backup_db_keep_last_n),
|
||||
("images", s.backup_images_keep_last_n),
|
||||
):
|
||||
candidates = session.execute(
|
||||
select(BackupRun)
|
||||
.where(BackupRun.kind == kind)
|
||||
.where(BackupRun.tag.is_(None))
|
||||
.where(BackupRun.status.in_(["ok", "error"]))
|
||||
.order_by(BackupRun.started_at.desc())
|
||||
.offset(keep)
|
||||
).scalars().all()
|
||||
for row in candidates:
|
||||
result = backup_service.unlink_artifact_files(
|
||||
sql_path=row.sql_path,
|
||||
tar_path=row.tar_path,
|
||||
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||
)
|
||||
counts["files_unlinked"] += sum(
|
||||
1 for v in result.values() if v
|
||||
)
|
||||
session.delete(row)
|
||||
counts[f"{kind}_deleted"] += 1
|
||||
session.commit()
|
||||
return counts
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.backup_db_nightly",
|
||||
soft_time_limit=60, time_limit=120,
|
||||
)
|
||||
def backup_db_nightly() -> dict:
|
||||
"""Hourly tick. Dispatches a real backup ONLY if the configured
|
||||
UTC hour matches and the nightly setting is enabled. Returns
|
||||
either {'skipped': '<reason>'} or {'dispatched': '<task_id>'}."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
s = session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
nightly_enabled = s.backup_db_nightly_enabled
|
||||
configured_hour = s.backup_db_nightly_hour_utc
|
||||
if not nightly_enabled:
|
||||
return {"skipped": "nightly disabled"}
|
||||
now_hour = datetime.now(UTC).hour
|
||||
if now_hour != configured_hour:
|
||||
return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
|
||||
res = backup_db_task.delay(triggered_by="nightly")
|
||||
return {"dispatched": res.id}
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from ..celery_app import celery
|
||||
@@ -27,7 +28,18 @@ def _async_session_factory():
|
||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.download.download_source", bind=True, acks_late=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.download.download_source",
|
||||
bind=True,
|
||||
acks_late=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=10,
|
||||
retry_backoff_max=120,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=900,
|
||||
time_limit=1200,
|
||||
)
|
||||
def download_source(self, source_id: int) -> int:
|
||||
"""Returns the DownloadEvent.id."""
|
||||
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
"""import_media_file task: imports one file via the Importer service and
|
||||
updates the ImportTask state machine + ImportBatch counters atomically.
|
||||
|
||||
Resilience contract (2026-05-24, operator-mandated): once a row has been
|
||||
flipped to 'processing' inside this task, EVERY exit path MUST flip it
|
||||
to a terminal state (complete / skipped / failed) or rely on Celery's
|
||||
autoretry to attempt the work again. No exit path is allowed to leave
|
||||
the row stuck in 'processing'. The `recover_interrupted_tasks`
|
||||
maintenance sweep is the safety net (5 min threshold) for the case
|
||||
where even the failure-marking commit can't be written.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImportBatch, ImportSettings, ImportTask
|
||||
@@ -29,9 +39,54 @@ def _map_result_to_status(result):
|
||||
return ("failed", False)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.import_file.import_media_file", bind=True)
|
||||
def _mark_failed(session, task, error_msg: str) -> None:
|
||||
"""Best-effort flip of a 'processing' row to 'failed' + batch counter
|
||||
increment. Wrapped in its own try because if the DB is what just
|
||||
broke, this commit will also fail — that's why the maintenance sweep
|
||||
exists as a backstop."""
|
||||
try:
|
||||
task.status = "failed"
|
||||
task.error = error_msg
|
||||
task.finished_at = datetime.now(UTC)
|
||||
session.add(task)
|
||||
session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == task.batch_id)
|
||||
.values(failed=ImportBatch.failed + 1)
|
||||
)
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001 — best-effort, sweep catches the rest
|
||||
try:
|
||||
session.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.import_file.import_media_file",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=300,
|
||||
time_limit=360,
|
||||
)
|
||||
def import_media_file(self, import_task_id: int) -> dict:
|
||||
"""Returns a dict so the eager-mode tests can assert without DB."""
|
||||
"""Returns a dict so the eager-mode tests can assert without DB.
|
||||
|
||||
Decorator notes:
|
||||
- autoretry_for: transient DB / filesystem errors retry with
|
||||
exponential backoff (5s base, jitter, max 3 attempts). On final
|
||||
give-up the task raises and acks_late=True (set globally on the
|
||||
Celery app) does NOT redeliver — the recovery sweep catches the
|
||||
row instead.
|
||||
- soft_time_limit (300s) raises SoftTimeLimitExceeded in this
|
||||
process so the task can mark its row failed before being killed.
|
||||
- time_limit (360s) is the hard cap; SIGKILL if the soft signal
|
||||
was swallowed.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
task = session.get(ImportTask, import_task_id)
|
||||
@@ -43,99 +98,104 @@ def import_media_file(self, import_task_id: int) -> dict:
|
||||
session.add(task)
|
||||
session.commit()
|
||||
|
||||
settings = session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
import_root = Path(settings.import_scan_path)
|
||||
batch = session.get(ImportBatch, task.batch_id)
|
||||
deep = bool(batch and batch.scan_mode == "deep")
|
||||
importer = Importer(
|
||||
session=session,
|
||||
images_root=IMAGES_ROOT,
|
||||
import_root=import_root,
|
||||
thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
|
||||
settings=settings,
|
||||
deep=deep,
|
||||
)
|
||||
|
||||
try:
|
||||
result = importer.import_one(Path(task.source_path))
|
||||
except Exception as exc: # pragma: no cover — pipeline crash
|
||||
task.status = "failed"
|
||||
task.error = f"{type(exc).__name__}: {exc}"
|
||||
task.finished_at = datetime.now(UTC)
|
||||
session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == task.batch_id)
|
||||
.values(failed=ImportBatch.failed + 1)
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
return _do_import(session, task, import_task_id)
|
||||
except SoftTimeLimitExceeded:
|
||||
_mark_failed(session, task, "soft_time_limit exceeded (>300s)")
|
||||
raise
|
||||
except (OperationalError, DBAPIError, OSError):
|
||||
# Retryable per the decorator; do NOT mark failed (let
|
||||
# autoretry have a clean go at it). If autoretry exhausts,
|
||||
# the row stays 'processing' and the maintenance sweep
|
||||
# flips it within 5 min.
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
||||
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
||||
raise
|
||||
|
||||
if result.status in ("imported", "superseded"):
|
||||
task.status = "complete"
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "imported"
|
||||
counter_col = ImportBatch.imported
|
||||
elif result.status == "attached":
|
||||
task.status = "complete"
|
||||
counter_col_name = "attachments"
|
||||
counter_col = ImportBatch.attachments
|
||||
elif result.status == "skipped":
|
||||
task.status = "skipped"
|
||||
task.error = (
|
||||
f"{result.skip_reason.value}: {result.error}"
|
||||
if result.skip_reason
|
||||
else result.error
|
||||
)
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "skipped"
|
||||
counter_col = ImportBatch.skipped
|
||||
else:
|
||||
task.status = "failed"
|
||||
task.error = result.error
|
||||
counter_col_name = "failed"
|
||||
counter_col = ImportBatch.failed
|
||||
|
||||
task.finished_at = datetime.now(UTC)
|
||||
def _do_import(session, task, import_task_id: int) -> dict:
|
||||
"""Actual work, called from inside the resilience wrapper."""
|
||||
settings = session.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
import_root = Path(settings.import_scan_path)
|
||||
batch = session.get(ImportBatch, task.batch_id)
|
||||
deep = bool(batch and batch.scan_mode == "deep")
|
||||
importer = Importer(
|
||||
session=session,
|
||||
images_root=IMAGES_ROOT,
|
||||
import_root=import_root,
|
||||
thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
|
||||
settings=settings,
|
||||
deep=deep,
|
||||
)
|
||||
|
||||
result = importer.import_one(Path(task.source_path))
|
||||
|
||||
if result.status in ("imported", "superseded"):
|
||||
task.status = "complete"
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "imported"
|
||||
counter_col = ImportBatch.imported
|
||||
elif result.status == "attached":
|
||||
task.status = "complete"
|
||||
counter_col_name = "attachments"
|
||||
counter_col = ImportBatch.attachments
|
||||
elif result.status == "skipped":
|
||||
task.status = "skipped"
|
||||
task.error = (
|
||||
f"{result.skip_reason.value}: {result.error}"
|
||||
if result.skip_reason
|
||||
else result.error
|
||||
)
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "skipped"
|
||||
counter_col = ImportBatch.skipped
|
||||
else:
|
||||
task.status = "failed"
|
||||
task.error = result.error
|
||||
counter_col_name = "failed"
|
||||
counter_col = ImportBatch.failed
|
||||
|
||||
task.finished_at = datetime.now(UTC)
|
||||
session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == task.batch_id)
|
||||
.values({counter_col_name: counter_col + 1})
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
|
||||
# Enqueue thumbnail + ML for newly imported AND superseded images
|
||||
# (a superseded row has cleared ML + no thumbnail).
|
||||
if result.status in ("imported", "superseded"):
|
||||
from .ml import tag_and_embed
|
||||
from .thumbnail import generate_thumbnail
|
||||
|
||||
ids = list(result.member_image_ids)
|
||||
if result.image_id is not None and result.image_id not in ids:
|
||||
ids.append(result.image_id)
|
||||
for img_id in ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
|
||||
# If this was the last task in the batch, mark the batch complete.
|
||||
remaining = session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.batch_id == task.batch_id)
|
||||
.where(ImportTask.status.in_(["pending", "queued", "processing"]))
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if remaining is None:
|
||||
session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == task.batch_id)
|
||||
.values({counter_col_name: counter_col + 1})
|
||||
.values(
|
||||
status="complete",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
|
||||
# Enqueue thumbnail + ML for newly imported AND superseded images
|
||||
# (a superseded row has cleared ML + no thumbnail).
|
||||
if result.status in ("imported", "superseded"):
|
||||
from .ml import tag_and_embed
|
||||
from .thumbnail import generate_thumbnail
|
||||
|
||||
ids = list(result.member_image_ids)
|
||||
if result.image_id is not None and result.image_id not in ids:
|
||||
ids.append(result.image_id)
|
||||
for img_id in ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
|
||||
# If this was the last task in the batch, mark the batch complete.
|
||||
remaining = session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.batch_id == task.batch_id)
|
||||
.where(ImportTask.status.in_(["pending", "queued", "processing"]))
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if remaining is None:
|
||||
session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == task.batch_id)
|
||||
.values(
|
||||
status="complete",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
return {"task_id": import_task_id, "status": task.status}
|
||||
return {"task_id": import_task_id, "status": task.status}
|
||||
|
||||
@@ -9,26 +9,32 @@ from PIL import Image
|
||||
from sqlalchemy import delete, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask
|
||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
||||
from ..utils.phash import compute_phash
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 30
|
||||
STUCK_THRESHOLD_MINUTES = 5
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
VERIFY_PAGE = 200
|
||||
FFPROBE_TIMEOUT_SECONDS = 10
|
||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
|
||||
"""Find ImportTask rows stuck in 'processing' for >5 min and re-queue them.
|
||||
|
||||
Why 30 min: large videos can legitimately take many minutes to import;
|
||||
30 is a safe gate that catches actual crashes (which leave the row stuck
|
||||
forever) without resetting slow-but-still-running jobs.
|
||||
Why 5 min: import_media_file is sub-second for the vast majority of
|
||||
files; even a large-video transcode caps at the per-task soft_time_limit
|
||||
(5 min) defined on the task itself. Anything still 'processing' after
|
||||
that window is a confirmed crash (worker died, DB disconnect mid-flush,
|
||||
OOM) and must be recycled. Was 30 min historically; tightened
|
||||
2026-05-24 after operator hit a 2224-row zombie pile during the IR
|
||||
migration scan.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
@@ -76,6 +82,71 @@ def cleanup_old_tasks() -> int:
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||
def recover_stalled_task_runs() -> int:
|
||||
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
||||
to 'error'. FC-3i.
|
||||
|
||||
A row gets stuck when the worker dies without emitting
|
||||
task_postrun / task_failure (e.g. OOM, container restart between
|
||||
signals, signal handler raised+logged). Shares the 5-min threshold
|
||||
with recover_interrupted_tasks for consistency.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
update(TaskRun)
|
||||
.where(TaskRun.status == "running")
|
||||
.where(TaskRun.started_at < cutoff)
|
||||
.values(
|
||||
status="error",
|
||||
error_type="RecoverySweep",
|
||||
error_message=(
|
||||
f"no completion signal received within "
|
||||
f"{STUCK_THRESHOLD_MINUTES} min"
|
||||
),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||
def prune_task_runs() -> dict:
|
||||
"""Daily retention for task_run rows. FC-3i.
|
||||
|
||||
- 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
|
||||
Success is high-volume, not interesting after a day.
|
||||
- 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
|
||||
(7 days default). Failures are operationally interesting longer.
|
||||
- 'running' rows: NEVER deleted by this task. The recovery sweep
|
||||
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||
terminal state; prune doesn't touch in-flight state.
|
||||
- 'retry' rows: treated as failures (>7d).
|
||||
|
||||
Returns dict of how many rows were deleted in each bucket.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||
with SessionLocal() as session:
|
||||
ok_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status == "ok")
|
||||
.where(TaskRun.finished_at < ok_cutoff)
|
||||
).rowcount or 0
|
||||
fail_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||
.where(TaskRun.finished_at < fail_cutoff)
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
||||
def backfill_phash() -> int:
|
||||
"""Recompute phash for stored images that have none (imported before
|
||||
|
||||
@@ -4,7 +4,8 @@ Dispatches to the right migrator based on `kind`. Updates MigrationRun
|
||||
row's status/counts/finished_at as it runs. Failures set status='error'
|
||||
with the error message preserved.
|
||||
|
||||
kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback, cleanup
|
||||
kinds: gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup
|
||||
(backup + rollback retired 2026-05-24 → see /api/system/backup/*)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -20,10 +21,8 @@ from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import MigrationRun
|
||||
from ..services.credential_crypto import CredentialCrypto
|
||||
from ..services.migrators import backup as backup_mod
|
||||
from ..services.migrators import cleanup as cleanup_mod
|
||||
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
||||
from ..services.migrators import rollback as rollback_mod
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -65,21 +64,11 @@ async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
||||
async with factory() as db:
|
||||
await _update_run(db, run_id, status="running")
|
||||
try:
|
||||
if kind == "backup":
|
||||
manifest = backup_mod.create_backup(
|
||||
db_url=get_config().database_url_sync,
|
||||
images_root=IMAGES_ROOT,
|
||||
tag=params.get("tag", "manual"),
|
||||
if kind in ("backup", "rollback"):
|
||||
raise ValueError(
|
||||
f"kind {kind!r} retired in FC-3h; "
|
||||
"use /api/system/backup/* instead"
|
||||
)
|
||||
await _update_run(
|
||||
db, run_id, status="ok",
|
||||
counts={"rows_processed": 0, "rows_inserted": 0,
|
||||
"rows_skipped": 0, "files_copied": 0,
|
||||
"bytes_copied": 0, "conflicts": 0},
|
||||
finished_at=datetime.now(UTC),
|
||||
metadata_patch={"manifest": manifest},
|
||||
)
|
||||
return manifest
|
||||
|
||||
elif kind == "gs_ingest":
|
||||
fc_crypto = CredentialCrypto(_KEY_PATH)
|
||||
@@ -166,21 +155,6 @@ async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
||||
)
|
||||
return result
|
||||
|
||||
elif kind == "rollback":
|
||||
result = rollback_mod.rollback_to_pre_migration(
|
||||
db_url=get_config().database_url_sync,
|
||||
images_root=IMAGES_ROOT,
|
||||
)
|
||||
await _update_run(
|
||||
db, run_id, status="ok",
|
||||
counts={"rows_processed": 0, "rows_inserted": 0,
|
||||
"rows_skipped": 0, "files_copied": 0,
|
||||
"bytes_copied": 0, "conflicts": 0},
|
||||
finished_at=datetime.now(UTC),
|
||||
metadata_patch={"rollback_result": result},
|
||||
)
|
||||
return result
|
||||
|
||||
else:
|
||||
raise ValueError(f"unknown kind: {kind}")
|
||||
|
||||
|
||||
+12
-1
@@ -9,6 +9,7 @@ apply_allowlist_tags sweeps which are 'maintenance' lane. Sync sessions
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImageRecord, MLSettings
|
||||
@@ -22,7 +23,17 @@ def _is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.tag_and_embed", bind=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.tag_and_embed",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=300,
|
||||
time_limit=420,
|
||||
)
|
||||
def tag_and_embed(self, image_id: int) -> dict:
|
||||
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
||||
then enqueue per-image allowlist application.
|
||||
|
||||
@@ -7,6 +7,8 @@ so they deserve their own queue lane.
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImageRecord
|
||||
from ..services.importer import is_video
|
||||
@@ -16,7 +18,17 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.thumbnail.generate_thumbnail", bind=True)
|
||||
@celery.task(
|
||||
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=120,
|
||||
time_limit=180,
|
||||
)
|
||||
def generate_thumbnail(self, image_id: int) -> dict:
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
node_modules/
|
||||
web-ext-artifacts/
|
||||
.web-ext-id
|
||||
.amo-upload-uuid
|
||||
@@ -0,0 +1,45 @@
|
||||
# FabledCurator Firefox Extension
|
||||
|
||||
Self-hosted Firefox extension that pushes session cookies from supported
|
||||
platforms (Patreon, SubscribeStar, Hentai-Foundry, Discord, Pixiv,
|
||||
DeviantArt) into FabledCurator, and lets you add a creator as a Source
|
||||
from their page in one click.
|
||||
|
||||
## Install (operator)
|
||||
|
||||
The signed XPI is bundled into the FC Docker image. Open FC →
|
||||
Settings → Maintenance → Browser extension → click "Install Firefox
|
||||
extension". Firefox shows its native install prompt. After installing,
|
||||
open the extension's options page (about:addons → FabledCurator →
|
||||
Preferences) and paste in the FC URL + extension API key shown on the
|
||||
same card.
|
||||
|
||||
## Develop
|
||||
|
||||
```sh
|
||||
cd extension/
|
||||
npm install --no-save # web-ext only
|
||||
npm run lint # web-ext lint
|
||||
npm run start # launches Firefox with extension loaded
|
||||
npm run build # unsigned XPI in web-ext-artifacts/
|
||||
```
|
||||
|
||||
## Smoke checklist (after every release that touches `extension/**`)
|
||||
|
||||
- [ ] `npm run lint` passes
|
||||
- [ ] `npm run start` loads the extension in a clean Firefox profile
|
||||
- [ ] Options page accepts FC URL + key, indicator turns green
|
||||
- [ ] Cookie export: log into patreon.com, click Patreon card → "X cookies exported"
|
||||
- [ ] Discord token: open discord.com, click Discord card → "Token captured"
|
||||
- [ ] Pixiv OAuth: click Pixiv card → login redirects, token stored
|
||||
- [ ] Add as source: visit patreon.com/<creator>, click floating button → toast
|
||||
- [ ] Subscriptions list: popup → "Sources" tab → list renders
|
||||
- [ ] Check now: click play icon on source row → no error toast
|
||||
|
||||
## Release
|
||||
|
||||
Bump `manifest.json` + `package.json` SemVer (both files) and commit
|
||||
under `extension/**`. The `.forgejo/workflows/extension.yml` workflow
|
||||
runs `web-ext sign` on main, commits the signed XPI to
|
||||
`frontend/public/extension/`, and the next FC server build bundles it
|
||||
into the Docker image.
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Background script — message router + Discord token capture
|
||||
* (webRequest) + Pixiv PKCE OAuth. Direct port of GS background.js;
|
||||
* api.js client points at FC instead of GS.
|
||||
*/
|
||||
|
||||
let discordToken = null;
|
||||
let discordTokenCapturedAt = null;
|
||||
|
||||
let pixivRefreshToken = null;
|
||||
let pixivTokenCapturedAt = null;
|
||||
let pixivOAuthPending = null;
|
||||
|
||||
const PIXIV_CLIENT_ID = 'MOBrBDS8blbauoSck0ZfDbtuzpyT';
|
||||
const PIXIV_CLIENT_SECRET = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj';
|
||||
const PIXIV_OAUTH_URL = 'https://app-api.pixiv.net/web/v1/login';
|
||||
const PIXIV_TOKEN_URL = 'https://oauth.secure.pixiv.net/auth/token';
|
||||
const PIXIV_REDIRECT_URI = 'https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback';
|
||||
|
||||
let initialized = false;
|
||||
|
||||
async function ensureInitialized() {
|
||||
if (initialized) return;
|
||||
await api.init();
|
||||
await loadDiscordToken();
|
||||
await loadPixivToken();
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
browser.runtime.onInstalled.addListener(() => ensureInitialized());
|
||||
browser.runtime.onStartup.addListener(() => ensureInitialized());
|
||||
ensureInitialized().catch(e => console.error('init failed:', e));
|
||||
|
||||
// ---- Discord token capture via webRequest ----
|
||||
|
||||
browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
(details) => {
|
||||
const auth = details.requestHeaders?.find(h => h.name.toLowerCase() === 'authorization');
|
||||
if (auth?.value && auth.value !== discordToken) {
|
||||
saveDiscordToken(auth.value);
|
||||
}
|
||||
},
|
||||
{ urls: ['https://discord.com/api/*'] },
|
||||
['requestHeaders'],
|
||||
);
|
||||
|
||||
async function loadDiscordToken() {
|
||||
const s = await browser.storage.local.get(['discordToken', 'discordTokenCapturedAt']);
|
||||
discordToken = s.discordToken || null;
|
||||
discordTokenCapturedAt = s.discordTokenCapturedAt || null;
|
||||
}
|
||||
|
||||
async function saveDiscordToken(token) {
|
||||
discordToken = token;
|
||||
discordTokenCapturedAt = new Date().toISOString();
|
||||
await browser.storage.local.set({ discordToken: token, discordTokenCapturedAt });
|
||||
}
|
||||
|
||||
// ---- Pixiv PKCE OAuth ----
|
||||
|
||||
async function loadPixivToken() {
|
||||
const s = await browser.storage.local.get(['pixivRefreshToken', 'pixivTokenCapturedAt']);
|
||||
pixivRefreshToken = s.pixivRefreshToken || null;
|
||||
pixivTokenCapturedAt = s.pixivTokenCapturedAt || null;
|
||||
}
|
||||
|
||||
async function savePixivToken(token) {
|
||||
pixivRefreshToken = token;
|
||||
pixivTokenCapturedAt = new Date().toISOString();
|
||||
await browser.storage.local.set({ pixivRefreshToken: token, pixivTokenCapturedAt });
|
||||
}
|
||||
|
||||
function generateCodeVerifier() {
|
||||
const a = new Uint8Array(32);
|
||||
crypto.getRandomValues(a);
|
||||
return base64UrlEncode(a);
|
||||
}
|
||||
|
||||
async function generateCodeChallenge(verifier) {
|
||||
const data = new TextEncoder().encode(verifier);
|
||||
const hash = await crypto.subtle.digest('SHA-256', data);
|
||||
return base64UrlEncode(new Uint8Array(hash));
|
||||
}
|
||||
|
||||
function base64UrlEncode(buf) {
|
||||
return btoa(String.fromCharCode(...buf)).replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
|
||||
}
|
||||
|
||||
async function initiatePixivOAuth() {
|
||||
const codeVerifier = generateCodeVerifier();
|
||||
const codeChallenge = await generateCodeChallenge(codeVerifier);
|
||||
|
||||
const params = new URLSearchParams({
|
||||
code_challenge: codeChallenge,
|
||||
code_challenge_method: 'S256',
|
||||
client: 'pixiv-android',
|
||||
});
|
||||
const tab = await browser.tabs.create({ url: `${PIXIV_OAUTH_URL}?${params}` });
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
pixivOAuthPending = { codeVerifier, tabId: tab.id, resolve, reject };
|
||||
setTimeout(() => {
|
||||
if (pixivOAuthPending && pixivOAuthPending.tabId === tab.id) {
|
||||
pixivOAuthPending = null;
|
||||
reject(new Error('Pixiv OAuth timed out (5 min)'));
|
||||
}
|
||||
}, 5 * 60 * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
browser.webRequest.onBeforeRedirect.addListener(
|
||||
async (details) => {
|
||||
if (!pixivOAuthPending) return;
|
||||
if (details.tabId !== pixivOAuthPending.tabId) return;
|
||||
const url = new URL(details.redirectUrl);
|
||||
const code = url.searchParams.get('code');
|
||||
if (!code) return;
|
||||
const verifier = pixivOAuthPending.codeVerifier;
|
||||
const resolve = pixivOAuthPending.resolve;
|
||||
const reject = pixivOAuthPending.reject;
|
||||
pixivOAuthPending = null;
|
||||
try {
|
||||
const tokenResp = await fetch(PIXIV_TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: new URLSearchParams({
|
||||
client_id: PIXIV_CLIENT_ID,
|
||||
client_secret: PIXIV_CLIENT_SECRET,
|
||||
code,
|
||||
code_verifier: verifier,
|
||||
grant_type: 'authorization_code',
|
||||
include_policy: 'true',
|
||||
redirect_uri: PIXIV_REDIRECT_URI,
|
||||
}),
|
||||
});
|
||||
const body = await tokenResp.json();
|
||||
if (!body.refresh_token) {
|
||||
reject(new Error(`Pixiv token exchange failed: ${JSON.stringify(body)}`));
|
||||
return;
|
||||
}
|
||||
await savePixivToken(body.refresh_token);
|
||||
try { await browser.tabs.remove(details.tabId); } catch {}
|
||||
resolve(body.refresh_token);
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
},
|
||||
{ urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] },
|
||||
);
|
||||
|
||||
// ---- Message router ----
|
||||
|
||||
browser.runtime.onMessage.addListener(async (msg) => {
|
||||
await ensureInitialized();
|
||||
switch (msg.type) {
|
||||
case 'GET_CONFIG':
|
||||
return { apiUrl: api.baseUrl, apiKey: api.apiKey };
|
||||
|
||||
case 'TEST_CONNECTION':
|
||||
try {
|
||||
await api.testConnection();
|
||||
return { connected: true };
|
||||
} catch (e) {
|
||||
return { connected: false, error: e.message };
|
||||
}
|
||||
|
||||
case 'GET_PLATFORM_STATUS': {
|
||||
const status = {};
|
||||
for (const key of Object.keys(PLATFORMS)) {
|
||||
if (PLATFORMS[key].authType === 'cookies') {
|
||||
status[key] = { hasCookies: false, cookieCount: 0 };
|
||||
try {
|
||||
const n = await getCookieCount(key);
|
||||
status[key] = { hasCookies: n > 0, cookieCount: n };
|
||||
} catch (e) {
|
||||
status[key] = { error: e.message };
|
||||
}
|
||||
} else if (key === 'discord') {
|
||||
status[key] = { hasToken: !!discordToken, capturedAt: discordTokenCapturedAt };
|
||||
} else if (key === 'pixiv') {
|
||||
status[key] = { hasToken: !!pixivRefreshToken, capturedAt: pixivTokenCapturedAt };
|
||||
} else {
|
||||
status[key] = {};
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
case 'EXPORT_COOKIES': {
|
||||
const key = msg.platform;
|
||||
const platform = PLATFORMS[key];
|
||||
if (!platform) return { error: `Unknown platform: ${key}` };
|
||||
try {
|
||||
if (platform.authType === 'cookies') {
|
||||
const cookies = await extractCookiesForPlatform(key);
|
||||
if (cookies.length === 0) return { error: 'No cookies found — log in first.' };
|
||||
const data = toNetscapeFormat(cookies);
|
||||
await api.uploadCredentials(key, 'cookies', data);
|
||||
return { success: true, cookieCount: cookies.length };
|
||||
}
|
||||
if (key === 'discord') {
|
||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||
await api.uploadCredentials('discord', 'token', discordToken);
|
||||
return { success: true };
|
||||
}
|
||||
if (key === 'pixiv') {
|
||||
if (!pixivRefreshToken) {
|
||||
await initiatePixivOAuth();
|
||||
}
|
||||
await api.uploadCredentials('pixiv', 'token', pixivRefreshToken);
|
||||
return { success: true };
|
||||
}
|
||||
return { error: 'Unsupported platform.' };
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
case 'EXPORT_ALL_COOKIES': {
|
||||
const results = {};
|
||||
for (const key of Object.keys(PLATFORMS)) {
|
||||
if (PLATFORMS[key].authType !== 'cookies') {
|
||||
results[key] = { skipped: true };
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const cookies = await extractCookiesForPlatform(key);
|
||||
if (cookies.length === 0) {
|
||||
results[key] = { skipped: true, reason: 'no cookies' };
|
||||
continue;
|
||||
}
|
||||
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
||||
results[key] = { success: true, cookieCount: cookies.length };
|
||||
} catch (e) {
|
||||
results[key] = { error: e.message };
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
case 'LIST_SOURCES':
|
||||
try {
|
||||
return { sources: await api.listSources() };
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'CHECK_SOURCE':
|
||||
try {
|
||||
return await api.triggerSourceCheck(msg.sourceId);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'ADD_AS_SOURCE':
|
||||
try {
|
||||
return await api.quickAddSource(msg.url);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
default:
|
||||
return { error: `Unknown message type: ${msg.type}` };
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
.fc-add-source-btn {
|
||||
all: revert;
|
||||
position: fixed; bottom: 24px; right: 24px; z-index: 2147483647;
|
||||
padding: 10px 16px; border-radius: 999px; border: none;
|
||||
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
||||
font: 500 14px/1.2 system-ui, sans-serif;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); cursor: pointer;
|
||||
transition: transform 100ms ease;
|
||||
}
|
||||
.fc-add-source-btn:hover { transform: translateY(-1px); }
|
||||
.fc-add-source-btn:disabled { opacity: 0.6; cursor: wait; }
|
||||
|
||||
.fc-toast {
|
||||
all: revert;
|
||||
position: fixed; bottom: 84px; right: 24px; z-index: 2147483647;
|
||||
max-width: 360px; padding: 12px 16px; border-radius: 8px;
|
||||
background: rgb(20, 23, 26); color: rgb(232, 228, 216);
|
||||
font: 14px/1.4 system-ui, sans-serif;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4);
|
||||
animation: fc-toast-in 200ms ease;
|
||||
}
|
||||
.fc-toast--success { border-left: 3px solid rgb(244, 186, 122); }
|
||||
.fc-toast--error { border-left: 3px solid rgb(220, 80, 80); }
|
||||
@keyframes fc-toast-in {
|
||||
from { transform: translateY(20px); opacity: 0; }
|
||||
to { transform: translateY(0); opacity: 1; }
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
(function () {
|
||||
if (window.__fc_addsource_injected) return;
|
||||
window.__fc_addsource_injected = true;
|
||||
|
||||
evaluate();
|
||||
|
||||
const reEval = () => evaluate();
|
||||
window.addEventListener('popstate', reEval);
|
||||
const origPush = history.pushState;
|
||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
||||
|
||||
function evaluate() {
|
||||
const platform = getPlatformFromUrl(window.location.href);
|
||||
const onArtist = platform && isArtistPage(window.location.href, platform);
|
||||
let btn = document.getElementById('fc-add-source-btn');
|
||||
if (onArtist && !btn) injectButton();
|
||||
else if (!onArtist && btn) btn.remove();
|
||||
}
|
||||
|
||||
function injectButton() {
|
||||
const btn = document.createElement('button');
|
||||
btn.id = 'fc-add-source-btn';
|
||||
btn.className = 'fc-add-source-btn';
|
||||
btn.textContent = '+ Add to FabledCurator';
|
||||
btn.addEventListener('click', onClick);
|
||||
document.body.appendChild(btn);
|
||||
}
|
||||
|
||||
async function onClick() {
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = 'Adding…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'ADD_AS_SOURCE',
|
||||
url: window.location.href,
|
||||
});
|
||||
if (r.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
} else {
|
||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||
}
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
}
|
||||
|
||||
function showToast(text, kind) {
|
||||
const t = document.createElement('div');
|
||||
t.className = `fc-toast fc-toast--${kind}`;
|
||||
t.textContent = text;
|
||||
document.body.appendChild(t);
|
||||
setTimeout(() => t.remove(), 4000);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,5 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 96 96" width="96" height="96">
|
||||
<rect width="96" height="96" rx="16" fill="#14171A"/>
|
||||
<text x="48" y="62" font-family="Georgia, serif" font-size="56" font-weight="500"
|
||||
fill="#F4BA7A" text-anchor="middle">F</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 284 B |
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* FC backend client. Talks to /api/credentials (FC-3b), /api/sources
|
||||
* (FC-3a), and the new /api/extension/* endpoints (FC-3g).
|
||||
*/
|
||||
|
||||
class FabledCuratorAPI {
|
||||
constructor() {
|
||||
this.baseUrl = null;
|
||||
this.apiKey = null;
|
||||
}
|
||||
|
||||
async init() {
|
||||
const cfg = await browser.storage.local.get(['apiUrl', 'apiKey']);
|
||||
this.baseUrl = cfg.apiUrl || null;
|
||||
this.apiKey = cfg.apiKey || null;
|
||||
return this.isConfigured();
|
||||
}
|
||||
|
||||
isConfigured() {
|
||||
return !!(this.baseUrl && this.apiKey);
|
||||
}
|
||||
|
||||
async request(method, endpoint, data = null) {
|
||||
if (!this.isConfigured()) {
|
||||
throw new Error('Not configured. Set FC URL + API key in settings.');
|
||||
}
|
||||
const url = `${this.baseUrl.replace(/\/+$/, '')}${endpoint}`;
|
||||
const options = {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Extension-Key': this.apiKey,
|
||||
},
|
||||
};
|
||||
if (data) options.body = JSON.stringify(data);
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, options);
|
||||
} catch (e) {
|
||||
throw new Error('Cannot connect to FabledCurator. Check URL.');
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
let message;
|
||||
try {
|
||||
const body = await response.json();
|
||||
message = body.error || `HTTP ${response.status}`;
|
||||
if (body.detail) message += `: ${body.detail}`;
|
||||
} catch {
|
||||
message = `HTTP ${response.status}: ${response.statusText}`;
|
||||
}
|
||||
const err = new Error(message);
|
||||
err.status = response.status;
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (response.status === 204) return { success: true };
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// FC-3b — credentials.
|
||||
uploadCredentials(platform, credentialType, data) {
|
||||
return this.request('POST', '/credentials', {
|
||||
platform,
|
||||
credential_type: credentialType,
|
||||
data,
|
||||
});
|
||||
}
|
||||
getCredentials() {
|
||||
return this.request('GET', '/credentials');
|
||||
}
|
||||
|
||||
// FC-3a — sources.
|
||||
listSources() {
|
||||
return this.request('GET', '/sources');
|
||||
}
|
||||
triggerSourceCheck(sourceId) {
|
||||
return this.request('POST', `/sources/${sourceId}/check`);
|
||||
}
|
||||
|
||||
// FC-3g — extension-specific.
|
||||
quickAddSource(url) {
|
||||
return this.request('POST', '/extension/quick-add-source', { url });
|
||||
}
|
||||
|
||||
// Connection test = the cheapest read with auth.
|
||||
testConnection() {
|
||||
return this.request('GET', '/credentials');
|
||||
}
|
||||
}
|
||||
|
||||
const api = new FabledCuratorAPI();
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* Cookie extraction + Netscape format conversion.
|
||||
* Direct port of GS extension/lib/cookies.js.
|
||||
*/
|
||||
|
||||
async function extractCookiesForPlatform(platformKey) {
|
||||
const platform = PLATFORMS[platformKey];
|
||||
if (!platform) throw new Error(`Unknown platform: ${platformKey}`);
|
||||
|
||||
const all = [];
|
||||
for (const domain of platform.domains) {
|
||||
try {
|
||||
const cookies = await browser.cookies.getAll({ domain });
|
||||
all.push(...cookies);
|
||||
} catch (e) {
|
||||
console.warn(`cookies.getAll failed for ${domain}:`, e);
|
||||
}
|
||||
}
|
||||
return deduplicateCookies(all);
|
||||
}
|
||||
|
||||
function deduplicateCookies(cookies) {
|
||||
const seen = new Map();
|
||||
for (const c of cookies) {
|
||||
const key = `${c.name}|${c.domain}|${c.path}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.set(key, c);
|
||||
} else {
|
||||
const existing = seen.get(key);
|
||||
if (c.expirationDate && (!existing.expirationDate || c.expirationDate > existing.expirationDate)) {
|
||||
seen.set(key, c);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(seen.values());
|
||||
}
|
||||
|
||||
function toNetscapeFormat(cookies) {
|
||||
const lines = ['# Netscape HTTP Cookie File'];
|
||||
for (const c of cookies) {
|
||||
let domain = c.domain.replace(/^\.?www\./, '.');
|
||||
if (!domain.startsWith('.')) domain = '.' + domain;
|
||||
const secure = c.secure ? 'TRUE' : 'FALSE';
|
||||
const expiration = c.expirationDate ? Math.floor(c.expirationDate) : 0;
|
||||
lines.push([domain, 'TRUE', c.path || '/', secure, String(expiration), c.name, c.value].join('\t'));
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function getCookieCount(platformKey) {
|
||||
try {
|
||||
return (await extractCookiesForPlatform(platformKey)).length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Platform definitions — mirrored from backend/app/services/platforms.py.
|
||||
* Two copies of these regexes (here and in backend extension_service.py)
|
||||
* is cheaper than building a shared definition file for two short
|
||||
* tables in different runtimes. Keep in sync by hand; reviewers catch
|
||||
* drift.
|
||||
*/
|
||||
|
||||
const PLATFORMS = {
|
||||
patreon: {
|
||||
name: 'Patreon',
|
||||
domains: ['.patreon.com', 'www.patreon.com', 'patreon.com'],
|
||||
authType: 'cookies',
|
||||
color: '#FF424D',
|
||||
urlPattern: /^https?:\/\/(www\.)?patreon\.com/,
|
||||
},
|
||||
subscribestar: {
|
||||
name: 'SubscribeStar',
|
||||
domains: [
|
||||
'.subscribestar.com',
|
||||
'.subscribestar.adult',
|
||||
'www.subscribestar.com',
|
||||
'subscribestar.com',
|
||||
'subscribestar.adult',
|
||||
],
|
||||
authType: 'cookies',
|
||||
color: '#FFD700',
|
||||
urlPattern: /^https?:\/\/(www\.)?subscribestar\.(com|adult)/,
|
||||
},
|
||||
hentaifoundry: {
|
||||
name: 'Hentai Foundry',
|
||||
domains: ['.hentai-foundry.com', 'www.hentai-foundry.com', 'hentai-foundry.com'],
|
||||
authType: 'cookies',
|
||||
color: '#9C27B0',
|
||||
urlPattern: /^https?:\/\/(www\.)?hentai-foundry\.com/,
|
||||
},
|
||||
discord: {
|
||||
name: 'Discord',
|
||||
domains: ['.discord.com', 'discord.com'],
|
||||
authType: 'token',
|
||||
color: '#5865F2',
|
||||
urlPattern: /^https?:\/\/(www\.)?discord\.com/,
|
||||
note: 'Open Discord in browser to capture token',
|
||||
},
|
||||
pixiv: {
|
||||
name: 'Pixiv',
|
||||
domains: ['.pixiv.net', 'www.pixiv.net', 'pixiv.net'],
|
||||
authType: 'token',
|
||||
color: '#0096FA',
|
||||
urlPattern: /^https?:\/\/(www\.)?pixiv\.net/,
|
||||
note: 'Click to authenticate via OAuth',
|
||||
},
|
||||
deviantart: {
|
||||
name: 'DeviantArt',
|
||||
domains: ['.deviantart.com', 'www.deviantart.com', 'deviantart.com'],
|
||||
authType: 'cookies',
|
||||
color: '#05CC47',
|
||||
urlPattern: /^https?:\/\/(www\.)?deviantart\.com/,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Per-platform artist-page URL patterns — these are STRICTER than
|
||||
* urlPattern (which matches any URL on the domain). Used by the content
|
||||
* script to decide whether to show the floating "Add as source" button.
|
||||
*/
|
||||
const PLATFORM_ARTIST_PATTERNS = {
|
||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c\/)[^/?#]+\/?$/i,
|
||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||
deviantart: /^https?:\/\/(www\.)?deviantart\.com\/(?!home$|watch\b|tag\b|browse\b)[^/?#]+\/?$/i,
|
||||
pixiv: /^https?:\/\/(www\.)?pixiv\.net\/(en\/)?users\/\d+/i,
|
||||
};
|
||||
|
||||
function getPlatformFromUrl(url) {
|
||||
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
||||
if (platform.urlPattern.test(url)) return key;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isArtistPage(url, platformKey) {
|
||||
const pattern = PLATFORM_ARTIST_PATTERNS[platformKey];
|
||||
return pattern ? pattern.test(url) : false;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.0",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
"gecko": {
|
||||
"id": "fabledcurator@fabledsword.com",
|
||||
"strict_min_version": "115.0"
|
||||
}
|
||||
},
|
||||
|
||||
"permissions": [
|
||||
"cookies",
|
||||
"storage",
|
||||
"tabs",
|
||||
"activeTab",
|
||||
"webRequest",
|
||||
"webRequestBlocking"
|
||||
],
|
||||
|
||||
"host_permissions": [
|
||||
"*://*.patreon.com/*",
|
||||
"*://*.subscribestar.com/*",
|
||||
"*://*.subscribestar.adult/*",
|
||||
"*://*.hentai-foundry.com/*",
|
||||
"*://*.discord.com/*",
|
||||
"*://*.pixiv.net/*",
|
||||
"*://*.deviantart.com/*",
|
||||
"*://app-api.pixiv.net/*",
|
||||
"*://oauth.secure.pixiv.net/*",
|
||||
"*://*/*"
|
||||
],
|
||||
|
||||
"action": {
|
||||
"default_popup": "popup/popup.html",
|
||||
"default_icon": "icons/icon.svg",
|
||||
"default_title": "FabledCurator"
|
||||
},
|
||||
|
||||
"background": {
|
||||
"scripts": ["lib/platforms.js", "lib/cookies.js", "lib/api.js", "background/background.js"]
|
||||
},
|
||||
|
||||
"options_ui": {
|
||||
"page": "options/options.html",
|
||||
"browser_style": true
|
||||
},
|
||||
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": [
|
||||
"*://*.patreon.com/*",
|
||||
"*://*.subscribestar.com/*",
|
||||
"*://*.subscribestar.adult/*",
|
||||
"*://*.hentai-foundry.com/*",
|
||||
"*://*.deviantart.com/*",
|
||||
"*://*.pixiv.net/*"
|
||||
],
|
||||
"js": ["lib/platforms.js", "content/content-script.js"],
|
||||
"css": ["content/content-script.css"],
|
||||
"run_at": "document_idle"
|
||||
}
|
||||
],
|
||||
|
||||
"icons": {
|
||||
"48": "icons/icon.svg",
|
||||
"96": "icons/icon.svg"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>FabledCurator — Settings</title>
|
||||
<style>
|
||||
body { font: 14px/1.4 system-ui, sans-serif; max-width: 520px; margin: 32px auto; padding: 0 16px; }
|
||||
h1 { font-family: Georgia, serif; font-size: 20px; }
|
||||
label { display: block; font-weight: 500; margin: 16px 0 6px; }
|
||||
input { width: 100%; padding: 8px; box-sizing: border-box; font: inherit; }
|
||||
.hint { color: #666; font-size: 12px; margin-top: 4px; }
|
||||
.row { display: flex; gap: 8px; margin-top: 16px; }
|
||||
button { padding: 8px 16px; cursor: pointer; font: inherit; }
|
||||
button.primary { background: #F4BA7A; color: #14171A; border: none; border-radius: 4px; }
|
||||
button.test { background: none; border: 1px solid #ccc; border-radius: 4px; }
|
||||
.status { margin-top: 16px; padding: 8px 12px; border-radius: 4px; }
|
||||
.status.ok { background: #DFF5DF; color: #2B6A2B; }
|
||||
.status.err { background: #FBE1E1; color: #862525; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>FabledCurator extension</h1>
|
||||
|
||||
<label for="api-url">FC base URL</label>
|
||||
<input id="api-url" type="url" placeholder="http://curator.example.com/api" />
|
||||
<div class="hint">Find this on FC → Settings → Maintenance → Browser extension.</div>
|
||||
|
||||
<label for="api-key">Extension API key</label>
|
||||
<input id="api-key" type="password" placeholder="paste from FC Settings card" />
|
||||
<div class="hint">Generate or rotate on FC → Settings → Maintenance → Browser extension.</div>
|
||||
|
||||
<div class="row">
|
||||
<button class="primary" id="save-btn">Save</button>
|
||||
<button class="test" id="test-btn">Test connection</button>
|
||||
</div>
|
||||
|
||||
<div id="status" class="status" style="display:none;"></div>
|
||||
|
||||
<script src="options.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,46 @@
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
const stored = await browser.storage.local.get(['apiUrl', 'apiKey']);
|
||||
document.getElementById('api-url').value = stored.apiUrl || '';
|
||||
document.getElementById('api-key').value = stored.apiKey || '';
|
||||
|
||||
document.getElementById('save-btn').addEventListener('click', save);
|
||||
document.getElementById('test-btn').addEventListener('click', test);
|
||||
});
|
||||
|
||||
async function save() {
|
||||
const apiUrl = document.getElementById('api-url').value.trim().replace(/\/+$/, '');
|
||||
const apiKey = document.getElementById('api-key').value.trim();
|
||||
if (!apiUrl || !apiKey) {
|
||||
showStatus('Both fields are required.', 'err');
|
||||
return;
|
||||
}
|
||||
await browser.storage.local.set({ apiUrl, apiKey });
|
||||
await browser.storage.local.remove(['lastConnectionTest', 'lastConnectionStatus']);
|
||||
showStatus('Saved.', 'ok');
|
||||
}
|
||||
|
||||
async function test() {
|
||||
const apiUrl = document.getElementById('api-url').value.trim().replace(/\/+$/, '');
|
||||
const apiKey = document.getElementById('api-key').value.trim();
|
||||
if (!apiUrl || !apiKey) {
|
||||
showStatus('Fill both fields first.', 'err');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const r = await fetch(`${apiUrl}/credentials`, {
|
||||
method: 'GET',
|
||||
headers: { 'X-Extension-Key': apiKey },
|
||||
});
|
||||
if (r.ok) showStatus(`Connected — HTTP ${r.status}.`, 'ok');
|
||||
else showStatus(`HTTP ${r.status}: ${r.statusText}`, 'err');
|
||||
} catch (e) {
|
||||
showStatus(`Cannot reach ${apiUrl}: ${e.message}`, 'err');
|
||||
}
|
||||
}
|
||||
|
||||
function showStatus(text, kind) {
|
||||
const el = document.getElementById('status');
|
||||
el.textContent = text;
|
||||
el.className = `status ${kind}`;
|
||||
el.style.display = 'block';
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
"lint": "web-ext lint --source-dir=.",
|
||||
"start": "web-ext run --source-dir=. --firefox=firefox",
|
||||
"build": "web-ext build --source-dir=. --overwrite-dest",
|
||||
"sign": "web-ext sign --source-dir=. --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
},
|
||||
"devDependencies": {
|
||||
"web-ext": "^8.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
:root {
|
||||
--bg: #14171A;
|
||||
--surface: #1F2428;
|
||||
--on-surface: #E8E4D8;
|
||||
--on-surface-variant: rgba(232, 228, 216, 0.65);
|
||||
--accent: #F4BA7A;
|
||||
--error: #DC5050;
|
||||
--success: #4CAF50;
|
||||
--warning: #FFB300;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 340px;
|
||||
font: 14px/1.4 system-ui, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--on-surface);
|
||||
}
|
||||
.hidden { display: none !important; }
|
||||
.topbar {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 10px 14px; background: var(--surface);
|
||||
}
|
||||
.brand { font-family: Georgia, serif; font-size: 16px; font-weight: 500; color: var(--accent); }
|
||||
.dot { width: 10px; height: 10px; border-radius: 50%; background: var(--on-surface-variant); }
|
||||
.dot.connected { background: var(--success); }
|
||||
|
||||
.setup { padding: 20px; text-align: center; }
|
||||
.alert {
|
||||
background: var(--surface); padding: 12px; border-radius: 6px;
|
||||
margin-bottom: 12px; border-left: 3px solid var(--accent);
|
||||
}
|
||||
.alert.alert-error { border-left-color: var(--error); }
|
||||
.alert strong { display: block; margin-bottom: 4px; }
|
||||
.alert p { margin: 0; color: var(--on-surface-variant); font-size: 13px; }
|
||||
|
||||
.tabs { display: flex; border-bottom: 1px solid var(--surface); }
|
||||
.tab {
|
||||
flex: 1; padding: 10px 0; background: none; border: none;
|
||||
color: var(--on-surface-variant); cursor: pointer; font: inherit;
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
.tab-panel { padding: 10px; }
|
||||
|
||||
.platform-card, .source-row {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 10px; margin-bottom: 6px; background: var(--surface);
|
||||
border-radius: 6px; cursor: pointer;
|
||||
}
|
||||
.platform-card.disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.platform-card.loading { opacity: 0.6; cursor: wait; }
|
||||
.platform-icon {
|
||||
width: 28px; height: 28px; border-radius: 50%;
|
||||
display: grid; place-items: center; font-weight: 600; color: white;
|
||||
}
|
||||
.info { flex: 1; min-width: 0; }
|
||||
.info .name { font-weight: 500; }
|
||||
.info .status { font-size: 12px; color: var(--on-surface-variant); }
|
||||
.info .status.ready { color: var(--success); }
|
||||
.info .status.error { color: var(--error); }
|
||||
.info .status.no-cookies { color: var(--warning); }
|
||||
.action-icon { color: var(--on-surface-variant); }
|
||||
|
||||
.btn {
|
||||
padding: 10px 14px; border: none; border-radius: 6px; cursor: pointer;
|
||||
font: 500 14px/1 system-ui, sans-serif;
|
||||
}
|
||||
.btn.primary { background: var(--accent); color: var(--bg); }
|
||||
.btn.primary:hover { filter: brightness(1.05); }
|
||||
.btn.primary:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.btn.block { display: block; width: 100%; margin-top: 8px; }
|
||||
.btn.link { background: none; color: var(--on-surface-variant); padding: 4px; }
|
||||
.btn.link:hover { color: var(--accent); }
|
||||
|
||||
.source-row .play {
|
||||
background: none; border: none; color: var(--on-surface-variant);
|
||||
cursor: pointer; padding: 4px;
|
||||
}
|
||||
.source-row .play:hover { color: var(--accent); }
|
||||
.source-row .url { font-size: 11px; color: var(--on-surface-variant); word-break: break-all; }
|
||||
|
||||
.status-message {
|
||||
padding: 10px 14px; margin: 8px 10px; border-radius: 6px;
|
||||
background: var(--surface); border-left: 3px solid;
|
||||
}
|
||||
.status-message.success { border-left-color: var(--success); }
|
||||
.status-message.error { border-left-color: var(--error); }
|
||||
.status-message.warning { border-left-color: var(--warning); }
|
||||
|
||||
.footer {
|
||||
border-top: 1px solid var(--surface);
|
||||
padding: 6px 10px; text-align: right;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="stylesheet" href="popup.css" />
|
||||
<title>FabledCurator</title>
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<span class="brand">FabledCurator</span>
|
||||
<span id="connection-status" class="dot" title="checking…"></span>
|
||||
</header>
|
||||
|
||||
<section id="setup-required" class="setup hidden">
|
||||
<div class="alert">
|
||||
<strong>Setup required</strong>
|
||||
<p>Configure the FC URL and extension API key.</p>
|
||||
</div>
|
||||
<button id="open-settings-btn" class="btn primary">Open settings</button>
|
||||
</section>
|
||||
|
||||
<section id="main-content" class="main hidden">
|
||||
<nav class="tabs">
|
||||
<button class="tab active" data-tab="platforms">Platforms</button>
|
||||
<button class="tab" data-tab="sources">Sources</button>
|
||||
</nav>
|
||||
|
||||
<div id="tab-platforms" class="tab-panel">
|
||||
<div id="platforms-list"></div>
|
||||
<button id="export-all-btn" class="btn primary block">Export all platforms</button>
|
||||
</div>
|
||||
|
||||
<div id="tab-sources" class="tab-panel hidden">
|
||||
<div id="sources-list"></div>
|
||||
</div>
|
||||
|
||||
<div id="status-message" class="status-message hidden"></div>
|
||||
|
||||
<footer class="footer">
|
||||
<button id="settings-btn" class="btn link">Settings</button>
|
||||
</footer>
|
||||
</section>
|
||||
|
||||
<script src="../lib/platforms.js"></script>
|
||||
<script src="popup.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,239 @@
|
||||
document.addEventListener('DOMContentLoaded', init);
|
||||
|
||||
const CONNECTION_TEST_INTERVAL = 2 * 60 * 1000;
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
const cfg = await browser.runtime.sendMessage({ type: 'GET_CONFIG' });
|
||||
if (!cfg || !cfg.apiUrl || !cfg.apiKey) {
|
||||
showSetupRequired();
|
||||
return;
|
||||
}
|
||||
document.getElementById('setup-required').classList.add('hidden');
|
||||
document.getElementById('main-content').classList.remove('hidden');
|
||||
setupEventListeners();
|
||||
showPlatformsLoading();
|
||||
testConnectionIfNeeded();
|
||||
loadPlatformStatus().catch(e => showError(`Failed to load platforms: ${e.message}`));
|
||||
} catch (e) {
|
||||
showSetupRequired();
|
||||
const alert = document.querySelector('#setup-required .alert');
|
||||
alert.textContent = '';
|
||||
const s = document.createElement('strong'); s.textContent = 'Error';
|
||||
const p = document.createElement('p'); p.textContent = e.message;
|
||||
alert.appendChild(s); alert.appendChild(p);
|
||||
alert.classList.add('alert-error');
|
||||
}
|
||||
}
|
||||
|
||||
function showSetupRequired() {
|
||||
document.getElementById('setup-required').classList.remove('hidden');
|
||||
document.getElementById('main-content').classList.add('hidden');
|
||||
document.getElementById('open-settings-btn').addEventListener('click', () => {
|
||||
browser.runtime.openOptionsPage();
|
||||
});
|
||||
}
|
||||
|
||||
function showPlatformsLoading() {
|
||||
const c = document.getElementById('platforms-list');
|
||||
c.textContent = '';
|
||||
const d = document.createElement('div');
|
||||
d.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||
d.textContent = 'Loading platforms…';
|
||||
c.appendChild(d);
|
||||
}
|
||||
|
||||
async function testConnectionIfNeeded() {
|
||||
const stored = await browser.storage.local.get(['lastConnectionTest', 'lastConnectionStatus']);
|
||||
const now = Date.now();
|
||||
if (now - (stored.lastConnectionTest || 0) < CONNECTION_TEST_INTERVAL && stored.lastConnectionStatus !== undefined) {
|
||||
updateConnectionDot(stored.lastConnectionStatus);
|
||||
if (!stored.lastConnectionStatus) showError('Cannot connect to backend (cached).');
|
||||
return;
|
||||
}
|
||||
const r = await browser.runtime.sendMessage({ type: 'TEST_CONNECTION' });
|
||||
await browser.storage.local.set({ lastConnectionTest: now, lastConnectionStatus: r.connected });
|
||||
updateConnectionDot(r.connected);
|
||||
if (!r.connected) showError(`Cannot connect to backend: ${r.error}`);
|
||||
}
|
||||
|
||||
function updateConnectionDot(connected) {
|
||||
const d = document.getElementById('connection-status');
|
||||
d.classList.toggle('connected', connected);
|
||||
d.title = connected ? 'Connected to FabledCurator' : 'Disconnected';
|
||||
}
|
||||
|
||||
async function loadPlatformStatus() {
|
||||
const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' });
|
||||
const c = document.getElementById('platforms-list');
|
||||
c.textContent = '';
|
||||
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
||||
c.appendChild(createPlatformCard(key, platform, status[key] || {}));
|
||||
}
|
||||
}
|
||||
|
||||
function createPlatformCard(key, platform, status) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'platform-card';
|
||||
card.dataset.platform = key;
|
||||
|
||||
const isTokenOnly = platform.authType === 'token' && !['discord', 'pixiv'].includes(key);
|
||||
const discordNeedsToken = key === 'discord' && !status.hasToken;
|
||||
if (isTokenOnly || discordNeedsToken) card.classList.add('disabled');
|
||||
|
||||
const icon = document.createElement('div');
|
||||
icon.className = 'platform-icon';
|
||||
icon.style.background = platform.color;
|
||||
icon.textContent = platform.name[0];
|
||||
|
||||
const info = document.createElement('div');
|
||||
info.className = 'info';
|
||||
const name = document.createElement('div');
|
||||
name.className = 'name';
|
||||
name.textContent = platform.name;
|
||||
const st = document.createElement('div');
|
||||
st.className = `status ${statusClass(status, platform, key)}`;
|
||||
st.textContent = statusText(status, platform, key);
|
||||
info.appendChild(name); info.appendChild(st);
|
||||
|
||||
const act = document.createElement('span');
|
||||
act.className = 'action-icon';
|
||||
act.textContent = '↥';
|
||||
|
||||
card.appendChild(icon); card.appendChild(info); card.appendChild(act);
|
||||
|
||||
if (!isTokenOnly && !discordNeedsToken) {
|
||||
card.addEventListener('click', () => exportPlatformCookies(key, card));
|
||||
}
|
||||
return card;
|
||||
}
|
||||
|
||||
function statusText(s, platform, key) {
|
||||
if (key === 'discord') return s.hasToken ? 'Token captured — ready' : 'Open Discord to capture token';
|
||||
if (key === 'pixiv') return s.hasToken ? 'Token captured — ready' : 'Click to authenticate via OAuth';
|
||||
if (platform.authType === 'token') return 'Manual token entry required';
|
||||
if (s.error) return 'Error checking cookies';
|
||||
if (!s.hasCookies || !s.cookieCount) return 'No cookies — log in first';
|
||||
return `${s.cookieCount} cookies ready`;
|
||||
}
|
||||
function statusClass(s, platform, key) {
|
||||
if (key === 'discord') return s.hasToken ? 'ready' : 'no-cookies';
|
||||
if (key === 'pixiv') return s.hasToken ? 'ready' : 'no-cookies';
|
||||
if (platform.authType === 'token') return 'no-cookies';
|
||||
if (s.error) return 'error';
|
||||
if (!s.hasCookies || !s.cookieCount) return 'no-cookies';
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
async function exportPlatformCookies(key, card) {
|
||||
card.classList.add('loading'); hideStatusMessage();
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key });
|
||||
if (r.error) showError(r.error);
|
||||
else {
|
||||
const n = r.cookieCount ?? null;
|
||||
const msg = n !== null
|
||||
? `${PLATFORMS[key].name}: ${n} cookies exported`
|
||||
: `${PLATFORMS[key].name}: token exported`;
|
||||
showSuccess(msg);
|
||||
await loadPlatformStatus();
|
||||
}
|
||||
} catch (e) { showError(e.message); }
|
||||
finally { card.classList.remove('loading'); }
|
||||
}
|
||||
|
||||
async function exportAllCookies() {
|
||||
const btn = document.getElementById('export-all-btn');
|
||||
btn.disabled = true; btn.textContent = 'Exporting…'; hideStatusMessage();
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'EXPORT_ALL_COOKIES' });
|
||||
const wins = Object.values(r).filter(x => x.success).length;
|
||||
const fails = Object.values(r).filter(x => !x.success && !x.skipped).length;
|
||||
if (wins && !fails) showSuccess(`Exported ${wins} platforms`);
|
||||
else if (wins) showWarning(`${wins} succeeded, ${fails} failed`);
|
||||
else if (fails) showError('All exports failed. Are you logged in?');
|
||||
else showWarning('Nothing to export');
|
||||
await loadPlatformStatus();
|
||||
} catch (e) { showError(e.message); }
|
||||
finally { btn.disabled = false; btn.textContent = 'Export all platforms'; }
|
||||
}
|
||||
|
||||
async function loadSources() {
|
||||
const c = document.getElementById('sources-list');
|
||||
c.textContent = '';
|
||||
const d = document.createElement('div');
|
||||
d.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||
d.textContent = 'Loading sources…';
|
||||
c.appendChild(d);
|
||||
const r = await browser.runtime.sendMessage({ type: 'LIST_SOURCES' });
|
||||
c.textContent = '';
|
||||
if (r.error) {
|
||||
const e = document.createElement('div');
|
||||
e.style.cssText = 'padding:12px;color:var(--error);';
|
||||
e.textContent = r.error;
|
||||
c.appendChild(e);
|
||||
return;
|
||||
}
|
||||
if (!r.sources || r.sources.length === 0) {
|
||||
const empty = document.createElement('div');
|
||||
empty.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||
empty.textContent = 'No sources yet.';
|
||||
c.appendChild(empty);
|
||||
return;
|
||||
}
|
||||
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
||||
}
|
||||
|
||||
function createSourceRow(src) {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'source-row';
|
||||
const info = document.createElement('div');
|
||||
info.className = 'info';
|
||||
const name = document.createElement('div');
|
||||
name.className = 'name';
|
||||
name.textContent = `${src.platform} · #${src.id}`;
|
||||
const url = document.createElement('div');
|
||||
url.className = 'url';
|
||||
url.textContent = src.url;
|
||||
info.appendChild(name); info.appendChild(url);
|
||||
const play = document.createElement('button');
|
||||
play.className = 'play';
|
||||
play.textContent = '▶';
|
||||
play.title = 'Check now';
|
||||
play.addEventListener('click', async () => {
|
||||
play.disabled = true;
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id });
|
||||
play.disabled = false;
|
||||
if (r.error) showError(r.error);
|
||||
else showSuccess(`Triggered check for source #${src.id}`);
|
||||
});
|
||||
row.appendChild(info); row.appendChild(play);
|
||||
return row;
|
||||
}
|
||||
|
||||
function setupEventListeners() {
|
||||
document.getElementById('export-all-btn').addEventListener('click', exportAllCookies);
|
||||
document.getElementById('settings-btn').addEventListener('click', () => browser.runtime.openOptionsPage());
|
||||
for (const tab of document.querySelectorAll('.tab')) {
|
||||
tab.addEventListener('click', () => {
|
||||
for (const t of document.querySelectorAll('.tab')) t.classList.remove('active');
|
||||
tab.classList.add('active');
|
||||
for (const p of document.querySelectorAll('.tab-panel')) p.classList.add('hidden');
|
||||
document.getElementById(`tab-${tab.dataset.tab}`).classList.remove('hidden');
|
||||
if (tab.dataset.tab === 'sources') loadSources();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function showSuccess(m) { showStatusMessage(m, 'success'); }
|
||||
function showError(m) { showStatusMessage(m, 'error'); }
|
||||
function showWarning(m) { showStatusMessage(m, 'warning'); }
|
||||
function showStatusMessage(text, kind) {
|
||||
const el = document.getElementById('status-message');
|
||||
el.textContent = text;
|
||||
el.className = `status-message ${kind}`;
|
||||
el.classList.remove('hidden');
|
||||
}
|
||||
function hideStatusMessage() {
|
||||
document.getElementById('status-message').classList.add('hidden');
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
sourceDir: '.',
|
||||
artifactsDir: './web-ext-artifacts',
|
||||
ignoreFiles: [
|
||||
'package.json',
|
||||
'package-lock.json',
|
||||
'web-ext-config.cjs',
|
||||
'web-ext-artifacts',
|
||||
'node_modules',
|
||||
'README.md',
|
||||
'.gitignore',
|
||||
],
|
||||
};
|
||||
@@ -18,5 +18,11 @@ const route = useRoute()
|
||||
<style scoped>
|
||||
.fc-content {
|
||||
min-height: 100vh;
|
||||
/* Push initial viewport content below the sticky TopNav. Without
|
||||
this, some views' first rows / form fields / table headers can
|
||||
end up obscured by the navbar (depending on parent overflow
|
||||
context interacting with position: sticky). Scrolled-down content
|
||||
still slides under the nav — the gradient-fade design is intact. */
|
||||
padding-top: 64px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
<template>
|
||||
<v-card class="fc-danger-zone mt-8" variant="outlined">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-alert-octagon" color="error" size="small" />
|
||||
<span>Danger zone</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 fc-muted mb-4">
|
||||
Cascade-delete this artist and every image, source, post, and
|
||||
attachment associated with them. This cannot be undone.
|
||||
Recoverable only from an FC-3h backup.
|
||||
</p>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete-forever"
|
||||
:loading="loading"
|
||||
@click="onClick"
|
||||
>Delete artist & cascade</v-btn>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-model="modalOpen"
|
||||
action="delete"
|
||||
kind="artist"
|
||||
:run-id="artistId"
|
||||
tier="C"
|
||||
:projected-counts="projectedCounts"
|
||||
:description="modalDescription"
|
||||
@confirm="onConfirm"
|
||||
/>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import { useAdminStore } from '../../stores/admin.js'
|
||||
|
||||
const props = defineProps({
|
||||
slug: { type: String, required: true },
|
||||
artistId: { type: Number, required: true },
|
||||
artistName: { type: String, required: true },
|
||||
})
|
||||
|
||||
const router = useRouter()
|
||||
const store = useAdminStore()
|
||||
const loading = ref(false)
|
||||
const modalOpen = ref(false)
|
||||
const projected = ref(null)
|
||||
|
||||
const projectedCounts = computed(() => projected.value?.projected || null)
|
||||
|
||||
const modalDescription = computed(
|
||||
() => projected.value
|
||||
? `Artist “${props.artistName}” — `
|
||||
+ `${projected.value.projected.images} images, `
|
||||
+ `${projected.value.projected.sources} sources, `
|
||||
+ `${Math.round(projected.value.projected.bytes_on_disk / 1_048_576)} MiB on disk`
|
||||
: '',
|
||||
)
|
||||
|
||||
async function onClick() {
|
||||
loading.value = true
|
||||
try {
|
||||
projected.value = await store.projectArtistCascade(props.slug)
|
||||
modalOpen.value = true
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onConfirm(token) {
|
||||
loading.value = true
|
||||
try {
|
||||
const result = await store.dispatchArtistCascade(props.slug, token)
|
||||
const taskId = result.task_id
|
||||
router.push('/artists')
|
||||
if (taskId) {
|
||||
store.pollTaskUntilDone(taskId).catch(() => {})
|
||||
}
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-danger-zone {
|
||||
border-color: rgb(var(--v-theme-error));
|
||||
border-radius: 8px;
|
||||
}
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -44,7 +44,31 @@
|
||||
</div>
|
||||
<div class="fc-tagcard__meta">
|
||||
<v-chip size="x-small" label>{{ card.kind }}</v-chip>
|
||||
<span class="fc-tagcard__count">{{ card.image_count }}</span>
|
||||
<div class="fc-tagcard__meta-right">
|
||||
<span class="fc-tagcard__count">{{ card.image_count }}</span>
|
||||
<v-menu>
|
||||
<template #activator="{ props: act }">
|
||||
<v-btn
|
||||
class="fc-tagcard__menu"
|
||||
icon="mdi-dots-vertical" size="x-small" variant="text"
|
||||
v-bind="act" @click.stop
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item
|
||||
title="Merge with…"
|
||||
prepend-icon="mdi-call-merge"
|
||||
@click="$emit('merge-with', card)"
|
||||
/>
|
||||
<v-list-item
|
||||
title="Delete tag"
|
||||
prepend-icon="mdi-delete"
|
||||
base-color="error"
|
||||
@click="$emit('delete', card)"
|
||||
/>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</div>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
@@ -54,7 +78,7 @@
|
||||
import { ref } from 'vue'
|
||||
|
||||
const props = defineProps({ card: { type: Object, required: true } })
|
||||
const emit = defineEmits(['open', 'rename', 'manage', 'read'])
|
||||
const emit = defineEmits(['open', 'rename', 'manage', 'read', 'merge-with', 'delete'])
|
||||
|
||||
const editing = ref(false)
|
||||
const draft = ref('')
|
||||
@@ -106,4 +130,13 @@ function submit() {
|
||||
}
|
||||
.fc-tagcard:hover .fc-tagcard__edit { opacity: .6; }
|
||||
.fc-tagcard__edit:hover { opacity: 1; }
|
||||
.fc-tagcard__meta-right {
|
||||
display: flex; align-items: center; gap: 4px;
|
||||
}
|
||||
.fc-tagcard__menu {
|
||||
opacity: 0;
|
||||
transition: opacity .15s ease;
|
||||
}
|
||||
.fc-tagcard:hover .fc-tagcard__menu { opacity: .6; }
|
||||
.fc-tagcard__menu:hover { opacity: 1; }
|
||||
</style>
|
||||
|
||||
@@ -55,19 +55,44 @@
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<div class="fc-bulk-panel__section">
|
||||
<h4>Destructive</h4>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill" block
|
||||
prepend-icon="mdi-delete-forever"
|
||||
:disabled="!sel.count"
|
||||
:loading="deleting"
|
||||
@click="onDeleteClick"
|
||||
>Delete {{ sel.count }} selected</v-btn>
|
||||
</div>
|
||||
|
||||
<div class="fc-bulk-panel__foot">
|
||||
<v-btn variant="text" block @click="sel.clear()">Clear selection</v-btn>
|
||||
</div>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-model="deleteModalOpen"
|
||||
action="delete"
|
||||
kind="images-selection"
|
||||
:run-id="bulkToken"
|
||||
tier="C"
|
||||
:projected-counts="bulkProjectedCounts"
|
||||
:description="bulkDescription"
|
||||
@confirm="onDeleteConfirm"
|
||||
/>
|
||||
</aside>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { useGallerySelectionStore } from '../../stores/gallerySelection.js'
|
||||
import { useAdminStore } from '../../stores/admin.js'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
|
||||
const sel = useGallerySelectionStore()
|
||||
const api = useApi()
|
||||
const adminStore = useAdminStore()
|
||||
|
||||
const addModel = ref(null)
|
||||
const addHits = ref([])
|
||||
@@ -99,6 +124,66 @@ async function onAddPick(id) {
|
||||
watch(() => sel.order.length, () => {
|
||||
if (sel.isSelectMode) sel.refresh()
|
||||
})
|
||||
|
||||
// --- FC-3k bulk delete -----------------------------------------------
|
||||
|
||||
const deleting = ref(false)
|
||||
const deleteModalOpen = ref(false)
|
||||
const bulkProjected = ref(null)
|
||||
const bulkToken = ref('')
|
||||
|
||||
const bulkProjectedCounts = computed(() => bulkProjected.value
|
||||
? {
|
||||
images: bulkProjected.value.images_found,
|
||||
thumbnails: bulkProjected.value.thumbs_to_unlink,
|
||||
bytes: bulkProjected.value.bytes_on_disk,
|
||||
}
|
||||
: null,
|
||||
)
|
||||
const bulkDescription = computed(
|
||||
() => bulkProjected.value
|
||||
? `${bulkProjected.value.images_found} images, `
|
||||
+ `${Math.round(bulkProjected.value.bytes_on_disk / 1_048_576)} MiB on disk`
|
||||
: '',
|
||||
)
|
||||
|
||||
async function _computeSha8(ids) {
|
||||
const canon = [...ids].sort((a, b) => a - b).join(',')
|
||||
const buf = new TextEncoder().encode(canon)
|
||||
const hashBuf = await crypto.subtle.digest('SHA-256', buf)
|
||||
const bytes = new Uint8Array(hashBuf)
|
||||
let hex = ''
|
||||
for (let i = 0; i < 4; i++) {
|
||||
hex += bytes[i].toString(16).padStart(2, '0')
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
async function onDeleteClick() {
|
||||
if (!sel.order.length) return
|
||||
deleting.value = true
|
||||
try {
|
||||
bulkProjected.value = await adminStore.projectBulkImageDelete(sel.order)
|
||||
bulkToken.value = await _computeSha8(sel.order)
|
||||
deleteModalOpen.value = true
|
||||
} finally {
|
||||
deleting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onDeleteConfirm(token) {
|
||||
deleting.value = true
|
||||
try {
|
||||
const result = await adminStore.dispatchBulkImageDelete(sel.order, token)
|
||||
const taskId = result.task_id
|
||||
if (taskId) {
|
||||
adminStore.pollTaskUntilDone(taskId).catch(() => {})
|
||||
}
|
||||
sel.clear()
|
||||
} finally {
|
||||
deleting.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
<template>
|
||||
<v-dialog
|
||||
:model-value="modelValue"
|
||||
max-width="520" persistent
|
||||
@update:model-value="$emit('update:modelValue', $event)"
|
||||
>
|
||||
<v-card>
|
||||
<v-card-title>
|
||||
{{ titleVerb }}
|
||||
{{ kindLabel }}<span v-if="runId"> #{{ runId }}</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<v-alert
|
||||
:type="alertType"
|
||||
variant="tonal" density="compact" class="mb-3"
|
||||
>
|
||||
<strong>{{ warningText }}</strong>
|
||||
<div v-if="description" class="mt-1">{{ description }}</div>
|
||||
</v-alert>
|
||||
|
||||
<div v-if="projectedCounts" class="fc-counts mb-3">
|
||||
<div v-for="(v, k) in projectedCounts" :key="k">
|
||||
<span class="fc-counts-key">{{ k }}:</span>
|
||||
<span class="fc-counts-val">{{ v }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template v-if="tier === 'C'">
|
||||
<div class="text-body-2 mb-2">Type the following to confirm:</div>
|
||||
<div class="fc-token mb-3">{{ expectedToken }}</div>
|
||||
<v-text-field
|
||||
v-model="typed"
|
||||
variant="outlined" density="compact" hide-details
|
||||
autofocus
|
||||
placeholder="paste the token above"
|
||||
/>
|
||||
</template>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="onCancel">Cancel</v-btn>
|
||||
<v-btn
|
||||
:color="confirmColor"
|
||||
variant="flat" rounded="pill"
|
||||
:disabled="!canConfirm"
|
||||
@click="onConfirm"
|
||||
>
|
||||
{{ titleVerb }}
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, required: true },
|
||||
action: { type: String, required: true }, // 'restore' | 'delete'
|
||||
kind: { type: String, required: true }, // 'db' | 'images' | 'artist' | 'tag' | 'images-selection'
|
||||
runId: { type: [Number, String], default: '' }, // numeric id or sha8 string
|
||||
description: { type: String, default: '' },
|
||||
tier: { type: String, default: 'C' }, // 'B' | 'C'
|
||||
projectedCounts: { type: Object, default: null },
|
||||
})
|
||||
const emit = defineEmits(['update:modelValue', 'confirm'])
|
||||
|
||||
const typed = ref('')
|
||||
const expectedToken = computed(
|
||||
() => `${props.action}-${props.kind}-${props.runId}`,
|
||||
)
|
||||
const titleVerb = computed(
|
||||
() => props.action === 'restore' ? 'Restore' : 'Delete',
|
||||
)
|
||||
const kindLabel = computed(() => ({
|
||||
db: 'database backup',
|
||||
images: 'images backup',
|
||||
artist: 'artist',
|
||||
tag: 'tag',
|
||||
'images-selection': 'image selection',
|
||||
}[props.kind] || props.kind))
|
||||
const alertType = computed(
|
||||
() => props.action === 'restore' ? 'warning' : 'error',
|
||||
)
|
||||
const confirmColor = computed(
|
||||
() => props.action === 'restore' ? 'warning' : 'error',
|
||||
)
|
||||
const warningText = computed(() => (
|
||||
props.action === 'restore'
|
||||
? 'This replaces current state with the backup. There is no undo.'
|
||||
: 'This permanently deletes the listed items. Cannot be recovered.'
|
||||
))
|
||||
const canConfirm = computed(
|
||||
() => props.tier === 'B'
|
||||
? true
|
||||
: typed.value === expectedToken.value,
|
||||
)
|
||||
|
||||
watch(() => props.modelValue, (open) => {
|
||||
if (open) typed.value = ''
|
||||
})
|
||||
|
||||
function onCancel() {
|
||||
emit('update:modelValue', false)
|
||||
}
|
||||
function onConfirm() {
|
||||
emit('confirm', expectedToken.value)
|
||||
emit('update:modelValue', false)
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-token {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
padding: 6px 10px; border-radius: 4px;
|
||||
font-size: 14px; word-break: break-all;
|
||||
}
|
||||
.fc-counts {
|
||||
display: grid; grid-template-columns: max-content auto;
|
||||
gap: 4px 12px;
|
||||
font-size: 13px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-counts-key { font-weight: 500; text-transform: capitalize; }
|
||||
.fc-counts-val { font-variant-numeric: tabular-nums; }
|
||||
</style>
|
||||
@@ -2,9 +2,6 @@
|
||||
<Teleport to="body">
|
||||
<div
|
||||
class="fc-viewer" role="dialog" aria-modal="true"
|
||||
@keydown.esc="$emit('close')"
|
||||
@keydown.left="onArrowLeft"
|
||||
@keydown.right="onArrowRight"
|
||||
tabindex="-1" ref="rootEl"
|
||||
>
|
||||
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
|
||||
@@ -68,7 +65,7 @@ import VideoCanvas from './VideoCanvas.vue'
|
||||
import TagPanel from './TagPanel.vue'
|
||||
import ProvenancePanel from './ProvenancePanel.vue'
|
||||
|
||||
defineEmits(['close'])
|
||||
const emit = defineEmits(['close'])
|
||||
|
||||
const modal = useModalStore()
|
||||
const rootEl = ref(null)
|
||||
@@ -86,14 +83,39 @@ const integrityBadge = computed(() => {
|
||||
})
|
||||
|
||||
let prevBodyOverflow = null
|
||||
|
||||
// Document-level keyboard handler. Bound on document (not on the modal
|
||||
// root) because <video> elements capture focus when the user interacts
|
||||
// with their controls — when video has focus, Escape and arrow keys
|
||||
// don't bubble reliably to ancestors. Listening on document side-steps
|
||||
// that. Filtered via isTextEntry so tag/comment inputs still get their
|
||||
// own keystrokes.
|
||||
function onKeyDown(ev) {
|
||||
if (ev.key === 'Escape') {
|
||||
if (isTextEntry(ev.target)) return
|
||||
ev.preventDefault()
|
||||
emit('close')
|
||||
} else if (ev.key === 'ArrowLeft') {
|
||||
if (isTextEntry(ev.target)) return
|
||||
ev.preventDefault()
|
||||
modal.goPrev()
|
||||
} else if (ev.key === 'ArrowRight') {
|
||||
if (isTextEntry(ev.target)) return
|
||||
ev.preventDefault()
|
||||
modal.goNext()
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
prevBodyOverflow = document.body.style.overflow
|
||||
document.body.style.overflow = 'hidden'
|
||||
document.addEventListener('keydown', onKeyDown, true)
|
||||
await nextFrame()
|
||||
rootEl.value?.focus()
|
||||
})
|
||||
onUnmounted(() => {
|
||||
document.body.style.overflow = prevBodyOverflow
|
||||
document.removeEventListener('keydown', onKeyDown, true)
|
||||
})
|
||||
|
||||
watch(() => modal.currentImageId, async () => {
|
||||
@@ -105,16 +127,6 @@ function nextFrame() {
|
||||
return new Promise(resolve => requestAnimationFrame(resolve))
|
||||
}
|
||||
|
||||
function onArrowLeft(ev) {
|
||||
if (isTextEntry(ev.target)) return
|
||||
ev.preventDefault()
|
||||
modal.goPrev()
|
||||
}
|
||||
function onArrowRight(ev) {
|
||||
if (isTextEntry(ev.target)) return
|
||||
ev.preventDefault()
|
||||
modal.goNext()
|
||||
}
|
||||
function isTextEntry(el) {
|
||||
if (!el) return false
|
||||
const tag = el.tagName
|
||||
|
||||
@@ -43,6 +43,21 @@
|
||||
class="fc-prov__desc" v-html="e.post.description_html"
|
||||
/>
|
||||
</article>
|
||||
|
||||
<!-- Fallback: image has artist_id set (folder-derived from
|
||||
filesystem import or set by tag_apply) but no ImageProvenance
|
||||
row exists. Show a minimal "by <artist>" line so the modal
|
||||
isn't blank for IR-imported images that aren't in any post. -->
|
||||
<article
|
||||
v-if="showArtistFallback"
|
||||
class="fc-prov__card"
|
||||
>
|
||||
<div class="fc-prov__meta">
|
||||
<RouterLink :to="`/artist/${fallbackArtist.slug}`">
|
||||
by {{ fallbackArtist.name }}
|
||||
</RouterLink>
|
||||
</div>
|
||||
</article>
|
||||
</template>
|
||||
|
||||
<div v-if="attachments.length" class="fc-prov__attach">
|
||||
@@ -79,13 +94,27 @@ const state = computed(() =>
|
||||
modal.currentImageId == null ? null : prov.imageProv(modal.currentImageId)
|
||||
)
|
||||
|
||||
// Show the panel while loading or on error, or when there is >=1 entry.
|
||||
// Hide entirely when the image has zero provenance.
|
||||
const fallbackArtist = computed(() => modal.current?.artist || null)
|
||||
|
||||
const showArtistFallback = computed(() => {
|
||||
const st = state.value
|
||||
// Only show the fallback when provenance is genuinely empty (no
|
||||
// entries, not loading, no error) AND the image has a direct
|
||||
// artist_id we can surface.
|
||||
if (!st || st.loading || st.error) return false
|
||||
if (Array.isArray(st.entries) && st.entries.length > 0) return false
|
||||
return fallbackArtist.value != null
|
||||
})
|
||||
|
||||
// Show the panel while loading or on error, when there is >=1 entry,
|
||||
// or when the artist fallback applies (image_record.artist_id set but
|
||||
// no ImageProvenance row, e.g. filesystem-imported IR images).
|
||||
const show = computed(() => {
|
||||
const st = state.value
|
||||
if (!st) return false
|
||||
if (st.loading || st.error) return true
|
||||
return Array.isArray(st.entries) && st.entries.length > 0
|
||||
if (Array.isArray(st.entries) && st.entries.length > 0) return true
|
||||
return showArtistFallback.value
|
||||
})
|
||||
|
||||
const attachments = computed(() => state.value?.attachments || [])
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
<template>
|
||||
<v-card class="fc-backup-card">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-database-export" size="small" />
|
||||
<span>Backups</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2 mb-4">
|
||||
pg_dump for the database (fast, nightly-schedulable);
|
||||
tar+zstd for the images (slow, manual only). Files live in
|
||||
<code>/images/_backups/</code>. Tag a backup to protect it
|
||||
from autoprune.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
v-if="store.lastError"
|
||||
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||
>
|
||||
{{ store.lastError }}
|
||||
</v-alert>
|
||||
|
||||
<!-- Database section -->
|
||||
<h3 class="fc-section-title">Database</h3>
|
||||
<div class="fc-settings-row mb-3">
|
||||
<v-switch
|
||||
:model-value="settings.backup_db_nightly_enabled"
|
||||
color="accent" density="compact" hide-details
|
||||
label="Nightly"
|
||||
@update:model-value="onSettingChange('backup_db_nightly_enabled', $event)"
|
||||
/>
|
||||
<v-text-field
|
||||
:model-value="settings.backup_db_nightly_hour_utc"
|
||||
label="Hour (UTC)" type="number" min="0" max="23"
|
||||
density="compact" hide-details style="max-width: 110px;"
|
||||
@update:model-value="onSettingChange('backup_db_nightly_hour_utc', Number($event))"
|
||||
/>
|
||||
<v-text-field
|
||||
:model-value="settings.backup_db_keep_last_n"
|
||||
label="Keep last" type="number" min="1" max="365"
|
||||
density="compact" hide-details style="max-width: 110px;"
|
||||
@update:model-value="onSettingChange('backup_db_keep_last_n', Number($event))"
|
||||
/>
|
||||
</div>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-database-arrow-up" class="mb-3"
|
||||
:loading="dbTriggering"
|
||||
@click="onTrigger('db')"
|
||||
>Run DB backup now</v-btn>
|
||||
|
||||
<BackupRunsTable
|
||||
:runs="store.dbRuns"
|
||||
@restore="onRestore"
|
||||
@delete="onDelete"
|
||||
@tag="onTag"
|
||||
/>
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<!-- Images section -->
|
||||
<h3 class="fc-section-title">Images</h3>
|
||||
<div class="fc-settings-row mb-3">
|
||||
<v-text-field
|
||||
:model-value="settings.backup_images_keep_last_n"
|
||||
label="Keep last" type="number" min="1" max="100"
|
||||
density="compact" hide-details style="max-width: 110px;"
|
||||
@update:model-value="onSettingChange('backup_images_keep_last_n', Number($event))"
|
||||
/>
|
||||
</div>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-folder-zip" class="mb-3"
|
||||
:loading="imagesTriggering"
|
||||
@click="onTrigger('images')"
|
||||
>Run images backup now</v-btn>
|
||||
<span class="text-caption fc-muted ml-2">
|
||||
ⓘ Hours for a 200GB+ library.
|
||||
</span>
|
||||
|
||||
<BackupRunsTable
|
||||
:runs="store.imagesRuns"
|
||||
@restore="onRestore"
|
||||
@delete="onDelete"
|
||||
@tag="onTag"
|
||||
/>
|
||||
</v-card-text>
|
||||
|
||||
<BackupConfirmModal
|
||||
v-model="confirmOpen"
|
||||
:action="confirmAction"
|
||||
:kind="confirmKind"
|
||||
:run-id="confirmRunId"
|
||||
:description="confirmDescription"
|
||||
@confirm="onConfirmSubmit"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import { useBackupStore } from '../../stores/backup.js'
|
||||
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import BackupRunsTable from './BackupRunsTable.vue'
|
||||
|
||||
const store = useBackupStore()
|
||||
|
||||
const settings = computed(() => store.settings || {
|
||||
backup_db_nightly_enabled: false,
|
||||
backup_db_nightly_hour_utc: 3,
|
||||
backup_db_keep_last_n: 14,
|
||||
backup_images_keep_last_n: 3,
|
||||
})
|
||||
|
||||
const dbTriggering = ref(false)
|
||||
const imagesTriggering = ref(false)
|
||||
const confirmOpen = ref(false)
|
||||
const confirmAction = ref('restore')
|
||||
const confirmKind = ref('db')
|
||||
const confirmRunId = ref(0)
|
||||
const confirmDescription = ref('')
|
||||
|
||||
let pollId = null
|
||||
function pollOnce() {
|
||||
if (document.hidden) return
|
||||
store.loadRuns('db')
|
||||
store.loadRuns('images')
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
store.loadSettings()
|
||||
pollOnce()
|
||||
pollId = setInterval(pollOnce, 5000)
|
||||
})
|
||||
onUnmounted(() => {
|
||||
if (pollId) { clearInterval(pollId); pollId = null }
|
||||
})
|
||||
|
||||
async function onTrigger(kind) {
|
||||
const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
|
||||
triggeringRef.value = true
|
||||
try {
|
||||
await store.triggerBackup(kind)
|
||||
pollOnce()
|
||||
} finally {
|
||||
triggeringRef.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onSettingChange(field, value) {
|
||||
await store.patchSettings({ [field]: value })
|
||||
}
|
||||
|
||||
function _openConfirm(action, run) {
|
||||
confirmAction.value = action
|
||||
confirmKind.value = run.kind
|
||||
confirmRunId.value = run.id
|
||||
confirmDescription.value = (
|
||||
action === 'restore'
|
||||
? `Source artifact: ${run.kind === 'db' ? run.sql_path : run.tar_path}`
|
||||
: 'Removes the BackupRun row and unlinks the artifact files.'
|
||||
)
|
||||
confirmOpen.value = true
|
||||
}
|
||||
|
||||
function onRestore(run) { _openConfirm('restore', run) }
|
||||
function onDelete(run) { _openConfirm('delete', run) }
|
||||
|
||||
async function onTag(run) {
|
||||
const next = prompt(
|
||||
`Tag for backup #${run.id} (blank to clear):`,
|
||||
run.tag || '',
|
||||
)
|
||||
if (next === null) return
|
||||
await store.setTag(run.id, next.trim() || null)
|
||||
pollOnce()
|
||||
}
|
||||
|
||||
async function onConfirmSubmit(token) {
|
||||
try {
|
||||
if (confirmAction.value === 'restore') {
|
||||
await store.restore(confirmRunId.value, token)
|
||||
} else {
|
||||
await store.deleteRun(confirmRunId.value, token)
|
||||
}
|
||||
pollOnce()
|
||||
} catch {
|
||||
// store.lastError already updated; surfaced in v-alert above.
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-backup-card { border-radius: 8px; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-section-title {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 16px; font-weight: 500;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.fc-settings-row {
|
||||
display: flex; gap: 12px; align-items: center; flex-wrap: wrap;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,109 @@
|
||||
<template>
|
||||
<v-table density="compact" class="fc-backup-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>When</th>
|
||||
<th>Status</th>
|
||||
<th class="text-right">Duration</th>
|
||||
<th class="text-right">Size</th>
|
||||
<th>Tag</th>
|
||||
<th class="text-right">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="r in runs" :key="r.id">
|
||||
<td class="fc-tabular" :title="r.started_at">
|
||||
{{ formatRelative(r.started_at) }}
|
||||
</td>
|
||||
<td>
|
||||
<v-icon size="small" :color="statusColor(r.status)">
|
||||
{{ statusIcon(r.status) }}
|
||||
</v-icon>
|
||||
{{ r.status }}
|
||||
</td>
|
||||
<td class="text-right fc-tabular">{{ formatDuration(r) }}</td>
|
||||
<td class="text-right fc-tabular">{{ formatBytes(r.size_bytes) }}</td>
|
||||
<td>
|
||||
<v-chip
|
||||
v-if="r.tag" size="x-small" color="accent" variant="tonal"
|
||||
>{{ r.tag }}</v-chip>
|
||||
<span v-else class="fc-muted">—</span>
|
||||
</td>
|
||||
<td class="text-right">
|
||||
<v-menu>
|
||||
<template #activator="{ props: act }">
|
||||
<v-btn icon="mdi-dots-vertical" size="x-small" variant="text" v-bind="act" />
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item
|
||||
:title="r.tag ? 'Untag' : 'Tag…'"
|
||||
@click="$emit('tag', r)"
|
||||
/>
|
||||
<v-list-item
|
||||
title="Restore…" :disabled="r.status !== 'ok'"
|
||||
@click="$emit('restore', r)"
|
||||
/>
|
||||
<v-list-item
|
||||
title="Delete…"
|
||||
@click="$emit('delete', r)"
|
||||
/>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</td>
|
||||
</tr>
|
||||
<tr v-if="!runs.length">
|
||||
<td colspan="6" class="text-center fc-muted py-4">No backups yet.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
defineProps({ runs: { type: Array, default: () => [] } })
|
||||
defineEmits(['restore', 'delete', 'tag'])
|
||||
|
||||
function statusIcon(s) {
|
||||
return {
|
||||
ok: 'mdi-check-circle', error: 'mdi-close-circle',
|
||||
running: 'mdi-timer-sand', restoring: 'mdi-restore-clock',
|
||||
restored: 'mdi-restore', pending: 'mdi-clock-outline',
|
||||
}[s] || 'mdi-help-circle'
|
||||
}
|
||||
function statusColor(s) {
|
||||
return {
|
||||
ok: 'success', error: 'error',
|
||||
running: 'accent', restoring: 'info',
|
||||
restored: 'info', pending: 'on-surface-variant',
|
||||
}[s] || 'on-surface-variant'
|
||||
}
|
||||
function formatDuration(r) {
|
||||
if (r.duration_seconds == null) return '—'
|
||||
const s = r.duration_seconds
|
||||
if (s < 60) return `${s}s`
|
||||
if (s < 3600) return `${Math.floor(s / 60)}m ${s % 60}s`
|
||||
const h = Math.floor(s / 3600); const m = Math.floor((s % 3600) / 60)
|
||||
return `${h}h ${m}m`
|
||||
}
|
||||
function formatBytes(b) {
|
||||
if (b == null) return '—'
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
|
||||
let i = 0; let v = b
|
||||
while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ }
|
||||
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
||||
}
|
||||
function formatRelative(iso) {
|
||||
if (!iso) return '—'
|
||||
const then = new Date(iso).getTime()
|
||||
const diff = Math.max(0, (Date.now() - then) / 1000)
|
||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||
return `${Math.floor(diff / 86400)}d ago`
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-backup-table { background: transparent; }
|
||||
.fc-tabular { font-variant-numeric: tabular-nums; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -0,0 +1,189 @@
|
||||
<template>
|
||||
<v-card class="fc-ext-card">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-puzzle" size="small" />
|
||||
<span>Browser extension</span>
|
||||
<span v-if="manifest?.installed" class="text-caption fc-muted">
|
||||
· Firefox · v{{ manifest.version }}
|
||||
</span>
|
||||
</v-card-title>
|
||||
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2">
|
||||
Pushes session cookies from supported platforms
|
||||
(patreon, subscribestar, hentaifoundry, discord, pixiv, deviantart)
|
||||
into FabledCurator, and lets you add a creator as a source from
|
||||
their page in one click.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
v-if="manifestError"
|
||||
type="warning" variant="tonal" density="compact" class="mt-3"
|
||||
>
|
||||
Could not load extension manifest: {{ manifestError }}
|
||||
</v-alert>
|
||||
|
||||
<v-alert
|
||||
v-else-if="manifest && !manifest.installed"
|
||||
type="warning" variant="tonal" density="compact" class="mt-3"
|
||||
>
|
||||
No bundled extension found in this image. Push a release that
|
||||
runs the extension CI workflow, or grab the XPI from the
|
||||
FabledCurator Forgejo releases page.
|
||||
</v-alert>
|
||||
|
||||
<template v-else-if="manifest?.installed">
|
||||
<div class="fc-ext-install mt-3">
|
||||
<v-btn
|
||||
v-if="isFirefox"
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-firefox"
|
||||
@click="installXpi"
|
||||
>Install Firefox extension</v-btn>
|
||||
|
||||
<v-btn
|
||||
variant="outlined" rounded="pill"
|
||||
:href="manifest.latest_url" download
|
||||
prepend-icon="mdi-download"
|
||||
>Download XPI</v-btn>
|
||||
|
||||
<v-alert
|
||||
v-if="!isFirefox"
|
||||
type="info" variant="tonal" density="compact" class="mt-3"
|
||||
>
|
||||
Open this page in Firefox to install in one click, or use
|
||||
"Download XPI" to install manually.
|
||||
</v-alert>
|
||||
</div>
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<div class="fc-muted text-body-2 mb-3">
|
||||
After installing, open the extension's options page
|
||||
(about:addons → FabledCurator → Preferences) and paste these:
|
||||
</div>
|
||||
|
||||
<v-text-field
|
||||
label="FC base URL" :model-value="apiUrl"
|
||||
readonly density="compact" hide-details
|
||||
append-inner-icon="mdi-content-copy"
|
||||
@click:append-inner="copy(apiUrl, 'URL')"
|
||||
/>
|
||||
|
||||
<v-text-field
|
||||
label="Extension API key"
|
||||
:model-value="apiKey"
|
||||
:type="keyShown ? 'text' : 'password'"
|
||||
readonly density="compact" hide-details class="mt-3"
|
||||
>
|
||||
<template #append-inner>
|
||||
<v-btn
|
||||
variant="text" density="compact" size="small"
|
||||
:icon="keyShown ? 'mdi-eye-off' : 'mdi-eye'"
|
||||
@click="keyShown = !keyShown"
|
||||
/>
|
||||
<v-btn
|
||||
variant="text" density="compact" size="small"
|
||||
icon="mdi-content-copy"
|
||||
@click="copy(apiKey, 'API key')"
|
||||
/>
|
||||
<v-btn
|
||||
variant="text" density="compact" size="small"
|
||||
icon="mdi-refresh" color="warning"
|
||||
:loading="rotating"
|
||||
@click="rotateKey"
|
||||
/>
|
||||
</template>
|
||||
</v-text-field>
|
||||
</template>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
|
||||
const api = useApi()
|
||||
|
||||
const manifest = ref(null)
|
||||
const manifestError = ref(null)
|
||||
const apiKey = ref('')
|
||||
const keyShown = ref(false)
|
||||
const rotating = ref(false)
|
||||
|
||||
const apiUrl = computed(() => `${window.location.origin}/api`)
|
||||
const isFirefox = computed(() => navigator.userAgent.includes('Firefox'))
|
||||
|
||||
onMounted(async () => {
|
||||
await Promise.all([loadManifest(), loadKey()])
|
||||
})
|
||||
|
||||
async function loadManifest() {
|
||||
try {
|
||||
manifest.value = await api.get('/api/extension/manifest')
|
||||
} catch (e) {
|
||||
if (e.status === 404) {
|
||||
// Backend says no XPI is bundled — surface the not-installed
|
||||
// state, not an error.
|
||||
manifest.value = { installed: false }
|
||||
} else {
|
||||
manifestError.value = e.message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function loadKey() {
|
||||
try {
|
||||
const { key } = await api.get('/api/settings/extension_api_key')
|
||||
apiKey.value = key
|
||||
} catch (e) {
|
||||
apiKey.value = ''
|
||||
window.__fcToast?.({
|
||||
text: `Failed to load extension API key: ${e.message}`,
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function installXpi() {
|
||||
if (!manifest.value?.latest_url) return
|
||||
window.location.assign(manifest.value.latest_url)
|
||||
}
|
||||
|
||||
async function rotateKey() {
|
||||
rotating.value = true
|
||||
try {
|
||||
const { key } = await api.post('/api/settings/extension_api_key/rotate')
|
||||
apiKey.value = key
|
||||
keyShown.value = true
|
||||
window.__fcToast?.({ text: 'Extension API key rotated.', type: 'success' })
|
||||
} catch (e) {
|
||||
window.__fcToast?.({
|
||||
text: `Rotate failed: ${e.message}`,
|
||||
type: 'error',
|
||||
})
|
||||
} finally {
|
||||
rotating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function copy(text, label) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text)
|
||||
window.__fcToast?.({ text: `${label} copied.`, type: 'success' })
|
||||
} catch (e) {
|
||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-ext-card { border-radius: 8px; }
|
||||
.fc-ext-install {
|
||||
display: flex; flex-wrap: wrap; gap: 8px;
|
||||
}
|
||||
.fc-muted {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
</style>
|
||||
@@ -12,6 +12,9 @@
|
||||
<MLThresholdSliders class="mt-4" />
|
||||
<AllowlistTable class="mt-4" />
|
||||
<AliasTable class="mt-4" />
|
||||
<BackupCard class="mt-6" />
|
||||
<TagMaintenanceCard class="mt-6" />
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
<LegacyMigrationCard class="mt-6" />
|
||||
</div>
|
||||
</template>
|
||||
@@ -22,6 +25,9 @@ import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
||||
import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||
import AllowlistTable from './AllowlistTable.vue'
|
||||
import AliasTable from './AliasTable.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import TagMaintenanceCard from './TagMaintenanceCard.vue'
|
||||
import BrowserExtensionCard from './BrowserExtensionCard.vue'
|
||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
||||
</script>
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
<template>
|
||||
<v-table density="compact" class="fc-queues-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Queue</th>
|
||||
<th class="text-right">Depth</th>
|
||||
<th class="text-right">Workers</th>
|
||||
<th v-if="!compact" class="text-right">Active</th>
|
||||
<th class="text-right">Last min: ok / err</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="q in QUEUE_NAMES" :key="q">
|
||||
<td>{{ q }}</td>
|
||||
<td class="text-right fc-tabular" :class="depthClass(q)">
|
||||
{{ formatDepth(q) }}
|
||||
</td>
|
||||
<td class="text-right fc-tabular">{{ workerCount(q) }}</td>
|
||||
<td v-if="!compact" class="text-right fc-tabular">{{ activeCount(q) }}</td>
|
||||
<td class="text-right fc-tabular">
|
||||
<span class="fc-ok">{{ recentOk(q) }}</span>
|
||||
<span class="fc-sep">/</span>
|
||||
<span :class="recentErr(q) > 0 ? 'fc-err' : 'fc-muted'">
|
||||
{{ recentErr(q) }}<span v-if="recentErr(q) > 0"> ⚠</span>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
|
||||
const props = defineProps({
|
||||
queues: { type: Object, default: null }, // store.queues
|
||||
workers: { type: Object, default: null }, // store.workers
|
||||
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
|
||||
compact: { type: Boolean, default: false },
|
||||
})
|
||||
|
||||
const QUEUE_NAMES = [
|
||||
'default', 'import', 'thumbnail', 'ml',
|
||||
'download', 'scan', 'maintenance',
|
||||
]
|
||||
|
||||
function formatDepth(name) {
|
||||
const d = props.queues?.queues?.[name]
|
||||
if (d == null) return '—'
|
||||
return d.toLocaleString()
|
||||
}
|
||||
|
||||
function depthClass(name) {
|
||||
const d = props.queues?.queues?.[name]
|
||||
const workers = workerCount(name)
|
||||
// Queue has depth but no workers → warning.
|
||||
if (d != null && d > 0 && workers === 0) return 'fc-warn'
|
||||
return ''
|
||||
}
|
||||
|
||||
function workerCount(name) {
|
||||
if (!props.workers?.workers) return 0
|
||||
let count = 0
|
||||
for (const info of Object.values(props.workers.workers)) {
|
||||
if (info.queues?.includes(name)) count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
function activeCount(name) {
|
||||
if (!props.workers?.workers) return 0
|
||||
// Active count is per-worker, not per-queue. Approximate: sum of
|
||||
// active_count across workers that subscribe to this queue.
|
||||
let count = 0
|
||||
for (const info of Object.values(props.workers.workers)) {
|
||||
if (info.queues?.includes(name)) count += info.active_count || 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
const recentByQueue = computed(() => {
|
||||
const out = {}
|
||||
for (const r of props.recentMinute) {
|
||||
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
||||
if (r.status === 'ok') out[r.queue].ok++
|
||||
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
function recentOk(name) { return recentByQueue.value[name]?.ok || 0 }
|
||||
function recentErr(name) { return recentByQueue.value[name]?.err || 0 }
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-queues-table { background: transparent; }
|
||||
.fc-tabular {
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-feature-settings: 'tnum';
|
||||
}
|
||||
.fc-ok { color: rgb(var(--v-theme-success, 76 175 80)); }
|
||||
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||
.fc-warn { color: rgb(var(--v-theme-warning, 255 179 0)); font-weight: 500; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-sep {
|
||||
margin: 0 4px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,64 @@
|
||||
<template>
|
||||
<v-card class="fc-activity-summary">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-pulse" size="small" />
|
||||
<span>System activity (last min)</span>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" size="small" rounded="pill"
|
||||
@click="$emit('open-activity')"
|
||||
>
|
||||
View activity
|
||||
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||
</v-btn>
|
||||
</v-card-title>
|
||||
|
||||
<v-card-text>
|
||||
<v-alert
|
||||
v-if="store.lastError"
|
||||
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||
>
|
||||
{{ store.lastError }}
|
||||
</v-alert>
|
||||
|
||||
<QueuesTable
|
||||
:queues="store.queues"
|
||||
:workers="store.workers"
|
||||
:recent-minute="store.recentMinute"
|
||||
compact
|
||||
/>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted } from 'vue'
|
||||
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
import QueuesTable from './QueuesTable.vue'
|
||||
|
||||
defineEmits(['open-activity'])
|
||||
|
||||
const store = useSystemActivityStore()
|
||||
|
||||
let pollId = null
|
||||
|
||||
function pollOnce() {
|
||||
if (document.hidden) return
|
||||
store.loadQueues()
|
||||
store.loadWorkers()
|
||||
store.loadRecentMinute()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
pollOnce()
|
||||
pollId = setInterval(pollOnce, 5000)
|
||||
})
|
||||
onUnmounted(() => {
|
||||
if (pollId) { clearInterval(pollId); pollId = null }
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-activity-summary { border-radius: 8px; }
|
||||
</style>
|
||||
@@ -0,0 +1,280 @@
|
||||
<template>
|
||||
<div class="fc-activity">
|
||||
<!-- Queues + workers pane -->
|
||||
<v-card class="mb-4">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-pulse" size="small" />
|
||||
<span>Queues + workers</span>
|
||||
<v-spacer />
|
||||
<span class="text-caption fc-muted">
|
||||
updated {{ formatRelative(store.queues?.fetched_at) }}
|
||||
</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<QueuesTable
|
||||
:queues="store.queues"
|
||||
:workers="store.workers"
|
||||
:recent-minute="store.recentMinute"
|
||||
/>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<!-- Recent failures pane -->
|
||||
<v-card class="mb-4">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-alert-circle-outline" size="small" />
|
||||
<span>Recent failures (last 24h)</span>
|
||||
<v-spacer />
|
||||
<span class="text-caption fc-muted">
|
||||
{{ failureCount }} failures
|
||||
</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<div v-if="errorTypes.length" class="mb-3 fc-pills">
|
||||
<v-chip
|
||||
v-for="t in errorTypes" :key="t.name"
|
||||
:color="t.name === filterErrorType ? 'accent' : undefined"
|
||||
size="small" variant="tonal" closable
|
||||
@click="toggleErrorTypeFilter(t.name)"
|
||||
@click:close="toggleErrorTypeFilter(t.name)"
|
||||
>
|
||||
{{ t.name }} × {{ t.count }}
|
||||
</v-chip>
|
||||
</div>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th>Queue</th>
|
||||
<th>Task</th>
|
||||
<th>Target</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="r in filteredFailures" :key="r.id">
|
||||
<td class="fc-tabular" :title="r.finished_at">
|
||||
{{ formatRelative(r.finished_at) }}
|
||||
</td>
|
||||
<td>{{ r.queue }}</td>
|
||||
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||
<td class="fc-err" :title="r.error_message">
|
||||
{{ r.error_type }}
|
||||
</td>
|
||||
</tr>
|
||||
<tr v-if="!filteredFailures.length">
|
||||
<td colspan="5" class="text-center fc-muted py-4">
|
||||
No failures in the last 24h.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<!-- All activity pane -->
|
||||
<v-card>
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-format-list-bulleted" size="small" />
|
||||
<span>All recent activity</span>
|
||||
<v-spacer />
|
||||
<v-select
|
||||
v-model="filterQueue"
|
||||
:items="queueOptions" density="compact" hide-details
|
||||
style="max-width: 180px;"
|
||||
@update:model-value="onFilterChange"
|
||||
/>
|
||||
<v-select
|
||||
v-model="filterStatus"
|
||||
:items="statusOptions" density="compact" hide-details
|
||||
style="max-width: 180px;"
|
||||
@update:model-value="onFilterChange"
|
||||
/>
|
||||
<v-btn
|
||||
variant="text" size="small" rounded="pill"
|
||||
@click="onRefresh"
|
||||
>
|
||||
<v-icon start size="small">mdi-refresh</v-icon>
|
||||
Refresh
|
||||
</v-btn>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Time</th>
|
||||
<th>Queue</th>
|
||||
<th>Task</th>
|
||||
<th>Target</th>
|
||||
<th class="text-right">Duration</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="r in store.runs" :key="r.id">
|
||||
<td><v-icon size="small" :color="statusColor(r.status)">{{ statusIcon(r.status) }}</v-icon></td>
|
||||
<td class="fc-tabular" :title="r.started_at">{{ formatRelative(r.started_at) }}</td>
|
||||
<td>{{ r.queue }}</td>
|
||||
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||
<td class="text-right fc-tabular">{{ formatDuration(r.duration_ms) }}</td>
|
||||
<td>{{ r.status }}</td>
|
||||
</tr>
|
||||
<tr v-if="!store.runs.length && !store.loading.runs">
|
||||
<td colspan="7" class="text-center fc-muted py-4">
|
||||
No activity yet.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
<div v-if="store.runsHasMore" class="d-flex justify-center py-3">
|
||||
<v-btn
|
||||
variant="text" size="small"
|
||||
:loading="store.loading.runs"
|
||||
@click="onLoadMore"
|
||||
>Load more</v-btn>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
import QueuesTable from './QueuesTable.vue'
|
||||
|
||||
const store = useSystemActivityStore()
|
||||
|
||||
const filterQueue = ref(null)
|
||||
const filterStatus = ref(null)
|
||||
const filterErrorType = ref(null)
|
||||
|
||||
const queueOptions = [
|
||||
{ title: 'All queues', value: null },
|
||||
{ title: 'import', value: 'import' },
|
||||
{ title: 'thumbnail', value: 'thumbnail' },
|
||||
{ title: 'ml', value: 'ml' },
|
||||
{ title: 'download', value: 'download' },
|
||||
{ title: 'scan', value: 'scan' },
|
||||
{ title: 'maintenance', value: 'maintenance' },
|
||||
{ title: 'default', value: 'default' },
|
||||
]
|
||||
const statusOptions = [
|
||||
{ title: 'All statuses', value: null },
|
||||
{ title: 'Running', value: 'running' },
|
||||
{ title: 'OK', value: 'ok' },
|
||||
{ title: 'Error', value: 'error' },
|
||||
{ title: 'Timeout', value: 'timeout' },
|
||||
{ title: 'Retry', value: 'retry' },
|
||||
]
|
||||
|
||||
let queuesPollId = null
|
||||
let failuresPollId = null
|
||||
|
||||
function pollQueues() {
|
||||
if (document.hidden) return
|
||||
store.loadQueues()
|
||||
store.loadWorkers()
|
||||
store.loadRecentMinute()
|
||||
}
|
||||
function pollFailures() {
|
||||
if (document.hidden) return
|
||||
store.loadFailures()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
pollQueues()
|
||||
pollFailures()
|
||||
store.setFilter({ queue: null, status: null })
|
||||
store.loadRuns({ reset: true })
|
||||
queuesPollId = setInterval(pollQueues, 3000)
|
||||
failuresPollId = setInterval(pollFailures, 15000)
|
||||
})
|
||||
onUnmounted(() => {
|
||||
if (queuesPollId) clearInterval(queuesPollId)
|
||||
if (failuresPollId) clearInterval(failuresPollId)
|
||||
})
|
||||
|
||||
const errorTypes = computed(() => {
|
||||
const counts = store.failures?.count_by_type || {}
|
||||
return Object.entries(counts).map(([name, count]) => ({ name, count }))
|
||||
})
|
||||
|
||||
const failureCount = computed(() => (store.failures?.recent || []).length)
|
||||
|
||||
const filteredFailures = computed(() => {
|
||||
const all = store.failures?.recent || []
|
||||
if (!filterErrorType.value) return all
|
||||
return all.filter(r => r.error_type === filterErrorType.value)
|
||||
})
|
||||
|
||||
function toggleErrorTypeFilter(name) {
|
||||
filterErrorType.value = filterErrorType.value === name ? null : name
|
||||
if (filterErrorType.value) {
|
||||
// Mirror into the All-activity table too.
|
||||
filterStatus.value = 'error'
|
||||
onFilterChange()
|
||||
}
|
||||
}
|
||||
|
||||
function onFilterChange() {
|
||||
store.setFilter({ queue: filterQueue.value, status: filterStatus.value })
|
||||
store.loadRuns({ reset: true })
|
||||
}
|
||||
function onLoadMore() { store.loadRuns({ reset: false }) }
|
||||
function onRefresh() { store.loadRuns({ reset: true }) }
|
||||
|
||||
function statusIcon(status) {
|
||||
return {
|
||||
ok: 'mdi-check-circle',
|
||||
error: 'mdi-close-circle',
|
||||
timeout: 'mdi-clock-alert',
|
||||
retry: 'mdi-refresh',
|
||||
running: 'mdi-timer-sand',
|
||||
}[status] || 'mdi-help-circle'
|
||||
}
|
||||
function statusColor(status) {
|
||||
return {
|
||||
ok: 'success', error: 'error', timeout: 'warning',
|
||||
retry: 'info', running: 'accent',
|
||||
}[status] || 'on-surface-variant'
|
||||
}
|
||||
function shortTaskName(name) {
|
||||
if (!name) return ''
|
||||
const parts = name.split('.')
|
||||
return parts.slice(-1)[0]
|
||||
}
|
||||
function formatDuration(ms) {
|
||||
if (ms == null) return '—'
|
||||
if (ms < 1000) return `${ms} ms`
|
||||
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
|
||||
return `${(ms / 60_000).toFixed(1)} min`
|
||||
}
|
||||
function formatRelative(iso) {
|
||||
if (!iso) return '—'
|
||||
const then = new Date(iso).getTime()
|
||||
const now = Date.now()
|
||||
const diff = Math.max(0, (now - then) / 1000)
|
||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||
return `${Math.floor(diff / 86400)}d ago`
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-activity { padding: 0; }
|
||||
.fc-pills {
|
||||
display: flex; gap: 6px; flex-wrap: wrap;
|
||||
}
|
||||
.fc-tabular {
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-feature-settings: 'tnum';
|
||||
}
|
||||
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -0,0 +1,95 @@
|
||||
<template>
|
||||
<v-card class="fc-tag-maint">
|
||||
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||
<v-icon icon="mdi-tag-remove" size="small" />
|
||||
<span>Tag maintenance</span>
|
||||
</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="fc-muted text-body-2 mb-4">
|
||||
Remove tags with zero image associations and zero series-page
|
||||
references. Auto-created tag rows that never got applied get
|
||||
swept here.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
v-if="store.lastError"
|
||||
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||
>{{ store.lastError }}</v-alert>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingPreview"
|
||||
class="mb-3"
|
||||
@click="onPreview"
|
||||
>Preview unused tags</v-btn>
|
||||
|
||||
<div v-if="preview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ preview.count }}</strong> unused tag(s).
|
||||
<span v-if="preview.count > 50" class="fc-muted">
|
||||
Showing first 50 names.
|
||||
</span>
|
||||
</p>
|
||||
<div v-if="preview.sample_names?.length" class="fc-name-grid mb-3">
|
||||
<span v-for="n in preview.sample_names" :key="n" class="fc-name">
|
||||
{{ n }}
|
||||
</span>
|
||||
</div>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete-sweep"
|
||||
:disabled="!preview.count"
|
||||
:loading="committing"
|
||||
@click="onCommit"
|
||||
>Delete {{ preview.count }} unused tag(s)</v-btn>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useAdminStore } from '../../stores/admin.js'
|
||||
|
||||
const store = useAdminStore()
|
||||
const preview = ref(null)
|
||||
const loadingPreview = ref(false)
|
||||
const committing = ref(false)
|
||||
|
||||
async function onPreview() {
|
||||
loadingPreview.value = true
|
||||
try {
|
||||
preview.value = await store.pruneUnusedTags({ dryRun: true })
|
||||
} finally {
|
||||
loadingPreview.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCommit() {
|
||||
committing.value = true
|
||||
try {
|
||||
const result = await store.pruneUnusedTags({ dryRun: false })
|
||||
preview.value = { count: 0, sample_names: result.sample_names || [] }
|
||||
} finally {
|
||||
committing.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-tag-maint { border-radius: 8px; }
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-name-grid {
|
||||
display: flex; flex-wrap: wrap; gap: 4px 8px;
|
||||
max-height: 200px; overflow-y: auto;
|
||||
padding: 8px; border-radius: 4px;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-name {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 12px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
</style>
|
||||
@@ -24,8 +24,16 @@ async function request(method, url, { body, params, signal } = {}) {
|
||||
const headers = {}
|
||||
const init = { method, headers, signal }
|
||||
if (body !== undefined) {
|
||||
headers['Content-Type'] = 'application/json'
|
||||
init.body = JSON.stringify(body)
|
||||
if (body instanceof FormData) {
|
||||
// Let the browser set Content-Type with the multipart boundary.
|
||||
// JSON-stringifying FormData produces "{}" and the backend sees
|
||||
// an empty multipart payload (this broke FC-5 IR/GS ingest from
|
||||
// the UI for several releases).
|
||||
init.body = body
|
||||
} else {
|
||||
headers['Content-Type'] = 'application/json'
|
||||
init.body = JSON.stringify(body)
|
||||
}
|
||||
}
|
||||
|
||||
const r = await fetch(fullUrl, init)
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
export const useAdminStore = defineStore('admin', () => {
|
||||
const api = useApi()
|
||||
const lastError = ref(null)
|
||||
|
||||
// --- Tier-C: artist cascade ---------------------------------------
|
||||
|
||||
async function projectArtistCascade(slug) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
`/api/admin/artists/${encodeURIComponent(slug)}/cascade-delete`,
|
||||
{ body: { dry_run: true } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function dispatchArtistCascade(slug, confirm) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
`/api/admin/artists/${encodeURIComponent(slug)}/cascade-delete`,
|
||||
{ body: { dry_run: false, confirm } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tier-C: bulk image delete ------------------------------------
|
||||
|
||||
async function projectBulkImageDelete(imageIds) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/images/bulk-delete',
|
||||
{ body: { image_ids: imageIds, dry_run: true } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function dispatchBulkImageDelete(imageIds, confirm) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/images/bulk-delete',
|
||||
{ body: { image_ids: imageIds, dry_run: false, confirm } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tier-B: tag delete + merge -----------------------------------
|
||||
|
||||
async function deleteTag(tagId) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.delete(`/api/admin/tags/${tagId}`)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function mergeTags(destId, sourceId) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
`/api/admin/tags/${destId}/merge`,
|
||||
{ body: { source_id: sourceId } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function tagUsageCount(tagId) {
|
||||
lastError.value = null
|
||||
try {
|
||||
const body = await api.get(`/api/admin/tags/${tagId}/usage-count`)
|
||||
return body.count
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tier-A: prune unused -----------------------------------------
|
||||
|
||||
async function pruneUnusedTags({ dryRun = true } = {}) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/tags/prune-unused',
|
||||
{ body: { dry_run: dryRun } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||
|
||||
/**
|
||||
* Polls /api/system/activity/runs?queue=maintenance every 3s,
|
||||
* resolves when a task_run row with the given celery task_id
|
||||
* reaches a terminal status (ok / error / timeout). Returns the
|
||||
* row. Times out after 30 min by default.
|
||||
*/
|
||||
async function pollTaskUntilDone(taskId, { timeoutMs = 1_800_000 } = {}) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const body = await api.get(
|
||||
'/api/system/activity/runs',
|
||||
{ params: { queue: 'maintenance', limit: 20 } },
|
||||
)
|
||||
const row = (body.runs || []).find(r => r.celery_task_id === taskId)
|
||||
if (row && ['ok', 'error', 'timeout'].includes(row.status)) {
|
||||
return row
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 3000))
|
||||
}
|
||||
throw new Error(`task ${taskId} did not finish within ${timeoutMs}ms`)
|
||||
}
|
||||
|
||||
return {
|
||||
lastError,
|
||||
projectArtistCascade,
|
||||
dispatchArtistCascade,
|
||||
projectBulkImageDelete,
|
||||
dispatchBulkImageDelete,
|
||||
deleteTag,
|
||||
mergeTags,
|
||||
tagUsageCount,
|
||||
pruneUnusedTags,
|
||||
pollTaskUntilDone,
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,109 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
export const useBackupStore = defineStore('backup', () => {
|
||||
const api = useApi()
|
||||
|
||||
const dbRuns = ref([])
|
||||
const imagesRuns = ref([])
|
||||
const settings = ref(null)
|
||||
const loading = ref({ dbRuns: false, imagesRuns: false, settings: false })
|
||||
const lastError = ref(null)
|
||||
|
||||
async function loadRuns(kind) {
|
||||
const targetRef = kind === 'db' ? dbRuns : imagesRuns
|
||||
const loadingKey = kind === 'db' ? 'dbRuns' : 'imagesRuns'
|
||||
loading.value[loadingKey] = true
|
||||
lastError.value = null
|
||||
try {
|
||||
const body = await api.get('/api/system/backup/runs', {
|
||||
params: { kind, limit: 50 },
|
||||
})
|
||||
targetRef.value = body.runs || []
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value[loadingKey] = false
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerBackup(kind, tag = null) {
|
||||
lastError.value = null
|
||||
try {
|
||||
await api.post(`/api/system/backup/${kind}`, {
|
||||
body: tag ? { tag } : {},
|
||||
})
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function restore(runId, confirm) {
|
||||
lastError.value = null
|
||||
try {
|
||||
await api.post(`/api/system/backup/runs/${runId}/restore`, {
|
||||
body: { confirm },
|
||||
})
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteRun(runId, confirm) {
|
||||
lastError.value = null
|
||||
try {
|
||||
await api.delete(`/api/system/backup/runs/${runId}`, {
|
||||
body: { confirm },
|
||||
})
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function setTag(runId, tag) {
|
||||
lastError.value = null
|
||||
try {
|
||||
await api.patch(`/api/system/backup/runs/${runId}`, {
|
||||
body: { tag },
|
||||
})
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSettings() {
|
||||
loading.value.settings = true
|
||||
lastError.value = null
|
||||
try {
|
||||
settings.value = await api.get('/api/system/backup/settings')
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.settings = false
|
||||
}
|
||||
}
|
||||
|
||||
async function patchSettings(patch) {
|
||||
lastError.value = null
|
||||
try {
|
||||
settings.value = await api.patch('/api/system/backup/settings', {
|
||||
body: patch,
|
||||
})
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dbRuns, imagesRuns, settings, loading, lastError,
|
||||
loadRuns, triggerBackup, restore, deleteRun, setTag,
|
||||
loadSettings, patchSettings,
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,103 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
const api = useApi()
|
||||
|
||||
// Live polled state.
|
||||
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
|
||||
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
|
||||
const recentMinute = ref([]) // last-60s rows (for Overview summary)
|
||||
const failures = ref(null) // { recent, count_by_type, since }
|
||||
|
||||
// Paginated runs (Activity tab "All recent activity" pane).
|
||||
const runs = ref([])
|
||||
const runsCursor = ref(null)
|
||||
const runsHasMore = ref(false)
|
||||
const runsFilter = ref({ queue: null, status: null, limit: 50 })
|
||||
|
||||
const loading = ref({ queues: false, workers: false, runs: false, failures: false })
|
||||
const lastError = ref(null)
|
||||
|
||||
async function loadQueues() {
|
||||
loading.value.queues = true
|
||||
lastError.value = null
|
||||
try {
|
||||
queues.value = await api.get('/api/system/activity/queues')
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.queues = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadWorkers() {
|
||||
loading.value.workers = true
|
||||
lastError.value = null
|
||||
try {
|
||||
workers.value = await api.get('/api/system/activity/workers')
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.workers = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRecentMinute() {
|
||||
// Used by the Overview summary card: pull last 60s of runs to compute
|
||||
// per-queue ok/err counts. One call covers all queues; UI groups.
|
||||
try {
|
||||
const body = await api.get('/api/system/activity/runs', {
|
||||
params: { limit: 200 },
|
||||
})
|
||||
recentMinute.value = body.runs || []
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRuns({ reset = false } = {}) {
|
||||
loading.value.runs = true
|
||||
lastError.value = null
|
||||
try {
|
||||
const params = { limit: runsFilter.value.limit }
|
||||
if (runsFilter.value.queue) params.queue = runsFilter.value.queue
|
||||
if (runsFilter.value.status) params.status = runsFilter.value.status
|
||||
if (!reset && runsCursor.value) params.before_id = runsCursor.value
|
||||
const body = await api.get('/api/system/activity/runs', { params })
|
||||
runs.value = reset ? body.runs : [...runs.value, ...body.runs]
|
||||
runsCursor.value = body.next_cursor
|
||||
runsHasMore.value = body.next_cursor !== null
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.runs = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadFailures() {
|
||||
loading.value.failures = true
|
||||
lastError.value = null
|
||||
try {
|
||||
failures.value = await api.get('/api/system/activity/failures')
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.failures = false
|
||||
}
|
||||
}
|
||||
|
||||
function setFilter(filter) {
|
||||
runsFilter.value = { ...runsFilter.value, ...filter }
|
||||
}
|
||||
|
||||
return {
|
||||
queues, workers, recentMinute, failures,
|
||||
runs, runsCursor, runsHasMore, runsFilter,
|
||||
loading, lastError,
|
||||
loadQueues, loadWorkers, loadRecentMinute,
|
||||
loadRuns, loadFailures, setFilter,
|
||||
}
|
||||
})
|
||||
@@ -96,6 +96,12 @@
|
||||
@open="openImage"
|
||||
/>
|
||||
</section>
|
||||
|
||||
<ArtistDangerZone
|
||||
:slug="slug"
|
||||
:artist-id="store.overview.id"
|
||||
:artist-name="store.overview.name"
|
||||
/>
|
||||
</template>
|
||||
</v-container>
|
||||
</template>
|
||||
@@ -106,6 +112,7 @@ import { useRoute, useRouter, RouterLink } from 'vue-router'
|
||||
import { useArtistStore } from '../stores/artist.js'
|
||||
import { useModalStore } from '../stores/modal.js'
|
||||
import MasonryGrid from '../components/discovery/MasonryGrid.vue'
|
||||
import ArtistDangerZone from '../components/artist/ArtistDangerZone.vue'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<v-container fluid class="py-6">
|
||||
<v-tabs v-model="tab" color="accent" class="mb-4">
|
||||
<v-tab value="overview">Overview</v-tab>
|
||||
<v-tab value="activity">Activity</v-tab>
|
||||
<v-tab value="import">Import</v-tab>
|
||||
<v-tab value="maintenance">Maintenance</v-tab>
|
||||
</v-tabs>
|
||||
@@ -9,6 +10,10 @@
|
||||
<v-window v-model="tab">
|
||||
<v-window-item value="overview">
|
||||
<SystemStatsCards :stats="system.stats" />
|
||||
<SystemActivitySummary
|
||||
class="mt-4"
|
||||
@open-activity="tab = 'activity'"
|
||||
/>
|
||||
<v-alert
|
||||
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
|
||||
type="info" variant="tonal" class="mt-4" closable
|
||||
@@ -18,6 +23,10 @@
|
||||
</v-alert>
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="activity">
|
||||
<SystemActivityTab />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="import">
|
||||
<ImportTriggerPanel />
|
||||
<v-divider class="my-6" />
|
||||
@@ -38,6 +47,8 @@ import { onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { useSystemStore } from '../stores/system.js'
|
||||
import { useImportStore } from '../stores/import.js'
|
||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
<TagCard
|
||||
v-for="c in store.cards" :key="c.id" :card="c"
|
||||
@open="openTag" @rename="onRename" @manage="onManage" @read="onRead"
|
||||
@merge-with="onMergeWith" @delete="onDeleteTag"
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -41,6 +42,49 @@
|
||||
@confirm="confirmMerge"
|
||||
@cancel="pendingMerge = null"
|
||||
/>
|
||||
|
||||
<v-dialog v-model="mergePickerOpen" max-width="520">
|
||||
<v-card>
|
||||
<v-card-title>Merge “{{ mergeSource?.name }}” into…</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 mb-3" style="opacity: 0.7;">
|
||||
Pick the target tag. Source tag will be deleted; all its
|
||||
image associations will move to the target. Must be same
|
||||
kind ({{ mergeSource?.kind }}).
|
||||
</p>
|
||||
<v-autocomplete
|
||||
v-model="mergeTargetId"
|
||||
:items="mergeHits" item-title="name" item-value="id"
|
||||
:loading="mergeLoading"
|
||||
density="compact" variant="outlined" hide-details
|
||||
placeholder="Search target tag…" no-filter
|
||||
@update:search="onMergeSearch"
|
||||
/>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="mergePickerOpen = false">Cancel</v-btn>
|
||||
<v-btn
|
||||
color="warning" variant="flat" rounded="pill"
|
||||
:disabled="!mergeTargetId"
|
||||
@click="onMergeConfirm"
|
||||
>Merge</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<DestructiveConfirmModal
|
||||
v-model="deleteTagModalOpen"
|
||||
action="delete"
|
||||
kind="tag"
|
||||
:run-id="deleteTagTarget?.id || 0"
|
||||
tier="B"
|
||||
:projected-counts="{ associations: deleteTagUsage }"
|
||||
:description="deleteTagTarget
|
||||
? `Delete tag “${deleteTagTarget.name}” and remove all image associations.`
|
||||
: ''"
|
||||
@confirm="onDeleteTagConfirm"
|
||||
/>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
@@ -48,8 +92,11 @@
|
||||
import { ref, watch, onMounted, onUnmounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { useTagDirectoryStore } from '../stores/tagDirectory.js'
|
||||
import { useAdminStore } from '../stores/admin.js'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
import TagCard from '../components/discovery/TagCard.vue'
|
||||
import MergeConfirmDialog from '../components/discovery/MergeConfirmDialog.vue'
|
||||
import DestructiveConfirmModal from '../components/modal/DestructiveConfirmModal.vue'
|
||||
|
||||
// Must stay a subset of the backend TagKind enum (character, fandom,
|
||||
// general, series, archive, post, meta, rating). 'fandom' is this
|
||||
@@ -106,6 +153,80 @@ function onManage(id) {
|
||||
function onRead(id) {
|
||||
router.push({ name: 'series-read', params: { tagId: id } })
|
||||
}
|
||||
|
||||
// --- FC-3k tag merge + delete ----------------------------------------
|
||||
|
||||
const adminStore = useAdminStore()
|
||||
const api = useApi()
|
||||
|
||||
// Tag merge via dots-menu (separate from inline-rename collision flow above)
|
||||
const mergePickerOpen = ref(false)
|
||||
const mergeSource = ref(null)
|
||||
const mergeTargetId = ref(null)
|
||||
const mergeHits = ref([])
|
||||
const mergeLoading = ref(false)
|
||||
let mergeDebounce = null
|
||||
|
||||
function onMergeWith(card) {
|
||||
mergeSource.value = card
|
||||
mergeTargetId.value = null
|
||||
mergeHits.value = []
|
||||
mergePickerOpen.value = true
|
||||
}
|
||||
|
||||
function onMergeSearch(q) {
|
||||
if (mergeDebounce) clearTimeout(mergeDebounce)
|
||||
if (!q) { mergeHits.value = []; return }
|
||||
mergeDebounce = setTimeout(async () => {
|
||||
mergeLoading.value = true
|
||||
try {
|
||||
const hits = await api.get('/api/tags/autocomplete', {
|
||||
params: { q, kind: mergeSource.value?.kind, limit: 20 },
|
||||
})
|
||||
mergeHits.value = (hits || []).filter(
|
||||
(h) => h.id !== mergeSource.value?.id,
|
||||
)
|
||||
} finally {
|
||||
mergeLoading.value = false
|
||||
}
|
||||
}, 250)
|
||||
}
|
||||
|
||||
async function onMergeConfirm() {
|
||||
if (!mergeSource.value || !mergeTargetId.value) return
|
||||
try {
|
||||
await adminStore.mergeTags(mergeTargetId.value, mergeSource.value.id)
|
||||
mergePickerOpen.value = false
|
||||
store.reset()
|
||||
} catch {
|
||||
// adminStore.lastError already set; UI surfaces it elsewhere.
|
||||
}
|
||||
}
|
||||
|
||||
// Tag delete via dots-menu (Tier B)
|
||||
const deleteTagModalOpen = ref(false)
|
||||
const deleteTagTarget = ref(null)
|
||||
const deleteTagUsage = ref(0)
|
||||
|
||||
async function onDeleteTag(card) {
|
||||
deleteTagTarget.value = card
|
||||
try {
|
||||
deleteTagUsage.value = await adminStore.tagUsageCount(card.id)
|
||||
} catch {
|
||||
deleteTagUsage.value = 0
|
||||
}
|
||||
deleteTagModalOpen.value = true
|
||||
}
|
||||
|
||||
async function onDeleteTagConfirm() {
|
||||
if (!deleteTagTarget.value) return
|
||||
try {
|
||||
await adminStore.deleteTag(deleteTagTarget.value.id)
|
||||
store.reset()
|
||||
} catch {
|
||||
// adminStore.lastError already set.
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
"""FC-3k: /api/admin/* endpoint integration tests.
|
||||
|
||||
Monkeypatch `.delay()` for Tier-C tasks to capture dispatch payloads
|
||||
without actually queuing. Tier-B/A endpoints run synchronously
|
||||
through the real service.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app import create_app
|
||||
from backend.app.models import Artist, ImageRecord, Tag, TagKind
|
||||
from backend.app.models.tag import image_tag
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def app():
|
||||
return create_app()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(app):
|
||||
async with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# --- Tier-C: POST /artists/<slug>/cascade-delete --------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_cascade_dry_run_returns_projection(client, db):
|
||||
a = Artist(name="Aria", slug="aria")
|
||||
db.add(a)
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/artists/aria/cascade-delete",
|
||||
json={"dry_run": True},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["artist"]["slug"] == "aria"
|
||||
assert "projected" in body
|
||||
assert body["projected"]["images"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_cascade_unknown_slug_404(client):
|
||||
resp = await client.post(
|
||||
"/api/admin/artists/no-such/cascade-delete",
|
||||
json={"dry_run": True},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_cascade_wrong_confirm_400(client, db):
|
||||
a = Artist(name="Aria", slug="aria")
|
||||
db.add(a)
|
||||
await db.commit()
|
||||
artist_id = a.id
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/artists/aria/cascade-delete",
|
||||
json={"dry_run": False, "confirm": "wrong"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "confirm_mismatch"
|
||||
assert body["expected"] == f"delete-artist-{artist_id}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_cascade_correct_confirm_dispatches(
|
||||
client, db, monkeypatch,
|
||||
):
|
||||
a = Artist(name="Aria", slug="aria")
|
||||
db.add(a)
|
||||
await db.commit()
|
||||
artist_id = a.id
|
||||
|
||||
dispatched = []
|
||||
|
||||
class _Result:
|
||||
id = "fake-task-id"
|
||||
|
||||
def _delay(**kw):
|
||||
dispatched.append(kw)
|
||||
return _Result()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.admin.delete_artist_cascade_task.delay", _delay,
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/artists/aria/cascade-delete",
|
||||
json={
|
||||
"dry_run": False,
|
||||
"confirm": f"delete-artist-{artist_id}",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
body = await resp.get_json()
|
||||
assert body["task_id"] == "fake-task-id"
|
||||
assert dispatched == [{"artist_id": artist_id}]
|
||||
|
||||
|
||||
# --- Tier-C: POST /images/bulk-delete -------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_dry_run_returns_counts(client, db, tmp_path):
|
||||
a = Artist(name="B", slug="b")
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
i1 = ImageRecord(
|
||||
artist_id=a.id, path=str(tmp_path / "1.jpg"),
|
||||
sha256="1" * 64, size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
i2 = ImageRecord(
|
||||
artist_id=a.id, path=str(tmp_path / "2.jpg"),
|
||||
sha256="2" * 64, size_bytes=20, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
db.add_all([i1, i2])
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/images/bulk-delete",
|
||||
json={
|
||||
"image_ids": [i1.id, i2.id, 9_999_999],
|
||||
"dry_run": True,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["images_found"] == 2
|
||||
assert body["bytes_on_disk"] == 30
|
||||
assert body["missing_ids"] == [9_999_999]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_empty_list_400(client):
|
||||
resp = await client.post(
|
||||
"/api/admin/images/bulk-delete",
|
||||
json={"image_ids": [], "dry_run": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "invalid_image_ids"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_non_int_400(client):
|
||||
resp = await client.post(
|
||||
"/api/admin/images/bulk-delete",
|
||||
json={"image_ids": ["foo", 2], "dry_run": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_wrong_confirm_400_with_expected_token(
|
||||
client, db, tmp_path,
|
||||
):
|
||||
a = Artist(name="B", slug="b")
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
img = ImageRecord(
|
||||
artist_id=a.id, path=str(tmp_path / "x.jpg"),
|
||||
sha256="c" * 64, size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
db.add(img)
|
||||
await db.commit()
|
||||
img_id = img.id
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/images/bulk-delete",
|
||||
json={
|
||||
"image_ids": [img_id], "dry_run": False, "confirm": "nope",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "confirm_mismatch"
|
||||
assert body["expected"].startswith("delete-images-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_correct_confirm_dispatches(
|
||||
client, db, tmp_path, monkeypatch,
|
||||
):
|
||||
a = Artist(name="B", slug="b")
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
img = ImageRecord(
|
||||
artist_id=a.id, path=str(tmp_path / "x.jpg"),
|
||||
sha256="d" * 64, size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
db.add(img)
|
||||
await db.commit()
|
||||
img_id = img.id
|
||||
|
||||
sha8 = hashlib.sha256(str(img_id).encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
dispatched = []
|
||||
|
||||
class _Result:
|
||||
id = "fake-bulk-id"
|
||||
|
||||
def _delay(**kw):
|
||||
dispatched.append(kw)
|
||||
return _Result()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.admin.bulk_delete_images_task.delay", _delay,
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/images/bulk-delete",
|
||||
json={
|
||||
"image_ids": [img_id],
|
||||
"dry_run": False,
|
||||
"confirm": f"delete-images-{sha8}",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
body = await resp.get_json()
|
||||
assert body["task_id"] == "fake-bulk-id"
|
||||
assert dispatched == [{"image_ids": [img_id]}]
|
||||
|
||||
|
||||
# --- Tier-B: DELETE /tags/<id> + POST /tags/<dest>/merge -----------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_delete_200_after_cascade(client, db):
|
||||
t = Tag(name="doomed-tag", kind=TagKind.general)
|
||||
db.add(t)
|
||||
await db.commit()
|
||||
tag_id = t.id
|
||||
|
||||
resp = await client.delete(f"/api/admin/tags/{tag_id}")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["deleted"]["id"] == tag_id
|
||||
assert body["associations_removed"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_delete_404_unknown(client):
|
||||
resp = await client.delete("/api/admin/tags/9999999")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_merge_succeeds_for_same_kind(client, db):
|
||||
src = Tag(name="src-tag", kind=TagKind.general)
|
||||
dest = Tag(name="dest-tag", kind=TagKind.general)
|
||||
db.add_all([src, dest])
|
||||
await db.commit()
|
||||
src_id = src.id
|
||||
dest_id = dest.id
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/admin/tags/{dest_id}/merge",
|
||||
json={"source_id": src_id},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["result"]["target_id"] == dest_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_merge_rejects_self_merge(client, db):
|
||||
t = Tag(name="x", kind=TagKind.general)
|
||||
db.add(t)
|
||||
await db.commit()
|
||||
resp = await client.post(
|
||||
f"/api/admin/tags/{t.id}/merge",
|
||||
json={"source_id": t.id},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_merge_returns_400_on_kind_mismatch(client, db):
|
||||
src = Tag(name="src", kind=TagKind.general)
|
||||
dest = Tag(name="dest", kind=TagKind.artist)
|
||||
db.add_all([src, dest])
|
||||
await db.commit()
|
||||
resp = await client.post(
|
||||
f"/api/admin/tags/{dest.id}/merge",
|
||||
json={"source_id": src.id},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "tag_kind_mismatch"
|
||||
|
||||
|
||||
# --- Tier-B helper: GET /tags/<id>/usage-count ----------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_usage_count_returns_zero_for_unused(client, db):
|
||||
t = Tag(name="lonely", kind=TagKind.general)
|
||||
db.add(t)
|
||||
await db.commit()
|
||||
resp = await client.get(f"/api/admin/tags/{t.id}/usage-count")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["count"] == 0
|
||||
|
||||
|
||||
# --- Tier-A: POST /tags/prune-unused --------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_unused_dry_run_lists_unused(client, db, tmp_path):
|
||||
a = Artist(name="P", slug="p")
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
img = ImageRecord(
|
||||
artist_id=a.id, path=str(tmp_path / "p.jpg"),
|
||||
sha256="e" * 64, size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
db.add(img)
|
||||
used = Tag(name="kept", kind=TagKind.general)
|
||||
unused = Tag(name="prune", kind=TagKind.general)
|
||||
db.add_all([used, unused])
|
||||
await db.flush()
|
||||
await db.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=used.id,
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/tags/prune-unused", json={"dry_run": True},
|
||||
)
|
||||
body = await resp.get_json()
|
||||
assert body["count"] == 1
|
||||
assert "prune" in body["sample_names"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_unused_commit_deletes_and_returns_count(client, db):
|
||||
db.add(Tag(name="byebye", kind=TagKind.general))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(
|
||||
"/api/admin/tags/prune-unused", json={"dry_run": False},
|
||||
)
|
||||
body = await resp.get_json()
|
||||
assert body["deleted"] >= 1
|
||||
assert "byebye" in body["sample_names"]
|
||||
@@ -0,0 +1,277 @@
|
||||
"""FC-3g: /api/extension and /extension/<filename> integration tests."""
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app import create_app
|
||||
from backend.app import frontend as frontend_module
|
||||
from backend.app.api import extension as extension_module
|
||||
from backend.app.models import AppSetting, Artist, Source
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def app():
|
||||
return create_app()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(app):
|
||||
async with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ext_key(db):
|
||||
db.add(AppSetting(key="extension_api_key", value="test-ext-key"))
|
||||
await db.commit()
|
||||
return "test-ext-key"
|
||||
|
||||
|
||||
# --- /api/extension/quick-add-source ---------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_creates_artist_and_source(client, ext_key, db_sync):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/maewix"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["created_artist"] is True
|
||||
assert body["created_source"] is True
|
||||
assert body["artist"]["slug"] == "maewix"
|
||||
assert body["source"]["platform"] == "patreon"
|
||||
assert body["source"]["enabled"] is True
|
||||
|
||||
# Async Core-DML assertion: column select, not ORM attribute access.
|
||||
artist_count = db_sync.execute(
|
||||
select(func.count(Artist.id)).where(Artist.slug == "maewix")
|
||||
).scalar_one()
|
||||
assert artist_count == 1
|
||||
source_count = db_sync.execute(
|
||||
select(func.count(Source.id)).where(
|
||||
Source.platform == "patreon",
|
||||
Source.url == "https://www.patreon.com/maewix",
|
||||
)
|
||||
).scalar_one()
|
||||
assert source_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_idempotent(client, ext_key):
|
||||
body = {"url": "https://www.subscribestar.com/some-creator"}
|
||||
headers = {"X-Extension-Key": ext_key}
|
||||
r1 = await client.post("/api/extension/quick-add-source", json=body, headers=headers)
|
||||
r2 = await client.post("/api/extension/quick-add-source", json=body, headers=headers)
|
||||
assert r1.status_code == 201
|
||||
assert r2.status_code == 200
|
||||
body2 = await r2.get_json()
|
||||
assert body2["created_source"] is False
|
||||
assert body2["created_artist"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url,platform,slug", [
|
||||
("https://www.patreon.com/maewix", "patreon", "maewix"),
|
||||
("https://patreon.com/maewix", "patreon", "maewix"),
|
||||
("https://www.subscribestar.com/foobar", "subscribestar", "foobar"),
|
||||
("https://subscribestar.adult/foobar", "subscribestar", "foobar"),
|
||||
("https://www.hentai-foundry.com/user/Foo/profile", "hentaifoundry", "Foo"),
|
||||
("https://www.deviantart.com/baz", "deviantart", "baz"),
|
||||
("https://www.pixiv.net/users/12345", "pixiv", "12345"),
|
||||
("https://www.pixiv.net/en/users/12345", "pixiv", "12345"),
|
||||
])
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_url_patterns(client, ext_key, url, platform, slug):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": url},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201, await resp.get_json()
|
||||
body = await resp.get_json()
|
||||
assert body["source"]["platform"] == platform
|
||||
# slugify lowercases — the Artist slug should reflect that.
|
||||
assert body["artist"]["slug"] == slug.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_unknown_url_400(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://example.com/foo"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "unknown_platform"
|
||||
assert "known" in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_invalid_url_400(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "not-a-url"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "invalid_url"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_missing_key_401(client):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/maewix"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "unauthorized"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_wrong_key_401(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/maewix"},
|
||||
headers={"X-Extension-Key": "wrong-key"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_missing_body_400(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "invalid_body"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_attaches_to_existing_artist(client, ext_key, db, db_sync):
|
||||
db.add(Artist(name="Maewix Original", slug="maewix", is_subscription=False))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/maewix"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["created_artist"] is False
|
||||
assert body["created_source"] is True
|
||||
|
||||
artist_count = db_sync.execute(
|
||||
select(func.count(Artist.id)).where(Artist.slug == "maewix")
|
||||
).scalar_one()
|
||||
assert artist_count == 1 # no duplicate created
|
||||
|
||||
|
||||
# --- /api/extension/manifest ---------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extension_manifest_returns_404_when_dir_missing(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(extension_module, "XPI_DIR", tmp_path / "does-not-exist")
|
||||
resp = await client.get("/api/extension/manifest")
|
||||
assert resp.status_code == 404
|
||||
body = await resp.get_json()
|
||||
assert body == {"installed": False}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extension_manifest_returns_404_when_no_xpi_files(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(extension_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/api/extension/manifest")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extension_manifest_returns_metadata_when_xpi_present(client, monkeypatch, tmp_path):
|
||||
xpi = tmp_path / "fabledcurator-1.2.3.xpi"
|
||||
xpi.write_bytes(b"fake-xpi-content")
|
||||
monkeypatch.setattr(extension_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/api/extension/manifest")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["installed"] is True
|
||||
assert body["version"] == "1.2.3"
|
||||
assert body["xpi_url"] == "/extension/fabledcurator-1.2.3.xpi"
|
||||
assert body["latest_url"] == "/extension/fabledcurator-latest.xpi"
|
||||
assert body["sha256"] == hashlib.sha256(b"fake-xpi-content").hexdigest()
|
||||
|
||||
|
||||
# --- /extension/<filename> -----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_rejects_non_xpi_filename(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/extension/passwd")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_rejects_path_traversal(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
# Path-traversal attempt — the route regex alone catches this since
|
||||
# `..` characters aren't in [\w.-]+, but cover the case anyway.
|
||||
resp = await client.get("/extension/fabledcurator-..%2Fetc%2Fpasswd.xpi")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_404_when_xpi_missing(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/extension/fabledcurator-1.0.0.xpi")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_serves_specific_xpi_with_correct_mime(
|
||||
client, monkeypatch, tmp_path,
|
||||
):
|
||||
xpi = tmp_path / "fabledcurator-1.0.0.xpi"
|
||||
xpi.write_bytes(b"xpi-bytes")
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/extension/fabledcurator-1.0.0.xpi")
|
||||
assert resp.status_code == 200
|
||||
assert resp.headers["Content-Type"].startswith("application/x-xpinstall")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_latest_returns_most_recent_xpi(
|
||||
client, monkeypatch, tmp_path,
|
||||
):
|
||||
import os
|
||||
import time
|
||||
|
||||
older = tmp_path / "fabledcurator-1.0.0.xpi"
|
||||
newer = tmp_path / "fabledcurator-1.0.1.xpi"
|
||||
older.write_bytes(b"old")
|
||||
newer.write_bytes(b"new")
|
||||
os.utime(older, (time.time() - 10, time.time() - 10))
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/extension/fabledcurator-latest.xpi")
|
||||
assert resp.status_code == 200
|
||||
data = await resp.get_data()
|
||||
assert data == b"new"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_extension_latest_404_when_dir_empty(client, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(frontend_module, "XPI_DIR", tmp_path)
|
||||
resp = await client.get("/extension/fabledcurator-latest.xpi")
|
||||
assert resp.status_code == 404
|
||||
+12
-37
@@ -31,17 +31,10 @@ async def client(app):
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_backup_returns_202_and_id(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate.run_migration",
|
||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
||||
)
|
||||
resp = await client.post("/api/migrate/backup", json={})
|
||||
assert resp.status_code == 202
|
||||
body = await resp.get_json()
|
||||
assert "run_id" in body
|
||||
assert body["status"] == "pending"
|
||||
# Retired 2026-05-24 (FC-3h): `test_post_backup_returns_202_and_id`
|
||||
# asserted the /api/migrate/backup endpoint, which was retired in FC-3h.
|
||||
# Backup is now a first-class feature at /api/system/backup/*;
|
||||
# coverage lives in tests/test_api_system_backup.py.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -62,10 +55,6 @@ async def test_post_ingest_rejects_missing_file(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate._has_recent_pre_migration_backup",
|
||||
lambda: True, # pretend backup exists
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate.run_migration",
|
||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
||||
@@ -83,32 +72,18 @@ async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
|
||||
assert resp.status_code == 202
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_apply_without_backup_rejected(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate._has_recent_pre_migration_backup",
|
||||
lambda: False,
|
||||
)
|
||||
export = {
|
||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
||||
"subscriptions": [], "credentials": [],
|
||||
}
|
||||
resp = await client.post(
|
||||
"/api/migrate/gs_ingest",
|
||||
form={"dry_run": "false"},
|
||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "no_backup"
|
||||
# Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
|
||||
# the migration backup-gate rejected applies without a recent backup.
|
||||
# That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
|
||||
# FC-3h will own backup as a first-class feature) so the test was
|
||||
# testing dead behavior. Removed rather than rewritten.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_dry_run_allowed_without_backup(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate._has_recent_pre_migration_backup",
|
||||
lambda: False,
|
||||
)
|
||||
# FC-3h: the _has_recent_pre_migration_backup gate was retired; dry-run
|
||||
# ingests were always allowed and now non-dry-run ingests are too.
|
||||
# This test stays as regression coverage that dry_run ingest still works.
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate.run_migration",
|
||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""FC-3i: /api/system/activity/* endpoint tests.
|
||||
|
||||
Mocks Redis LLEN + celery inspect via monkeypatch on the module-level
|
||||
_read_queues_sync / _read_workers_sync helpers, so tests don't need
|
||||
a live broker or running workers.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from backend.app import create_app
|
||||
from backend.app.api import system_activity as activity_module
|
||||
from backend.app.models import TaskRun
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def app():
|
||||
return create_app()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(app):
|
||||
async with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_caches(monkeypatch):
|
||||
"""Clear the module-level caches between tests so cache state from
|
||||
one test doesn't leak into the next."""
|
||||
monkeypatch.setitem(activity_module._QUEUE_CACHE, "data", None)
|
||||
monkeypatch.setitem(activity_module._QUEUE_CACHE, "ts", 0.0)
|
||||
monkeypatch.setitem(activity_module._WORKER_CACHE, "data", None)
|
||||
monkeypatch.setitem(activity_module._WORKER_CACHE, "ts", 0.0)
|
||||
|
||||
|
||||
# --- /queues -------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queues_returns_all_known_queues(client, monkeypatch):
|
||||
def _fake():
|
||||
return {
|
||||
"queues": dict.fromkeys(activity_module._QUEUE_NAMES, 0),
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
monkeypatch.setattr(activity_module, "_read_queues_sync", _fake)
|
||||
|
||||
resp = await client.get("/api/system/activity/queues")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert set(body["queues"].keys()) == set(activity_module._QUEUE_NAMES)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queues_cached_within_ttl(client, monkeypatch):
|
||||
call_count = {"n": 0}
|
||||
|
||||
def _counting():
|
||||
call_count["n"] += 1
|
||||
return {"queues": {}, "fetched_at": "x"}
|
||||
monkeypatch.setattr(activity_module, "_read_queues_sync", _counting)
|
||||
|
||||
await client.get("/api/system/activity/queues")
|
||||
await client.get("/api/system/activity/queues")
|
||||
await client.get("/api/system/activity/queues")
|
||||
assert call_count["n"] == 1 # cached after the first call
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queues_redis_failure_returns_null_per_failing_queue(client, monkeypatch):
|
||||
"""_read_queues_sync's per-queue try/except returns None on failure."""
|
||||
def _partial():
|
||||
out = dict.fromkeys(activity_module._QUEUE_NAMES, 0)
|
||||
out["ml"] = None # simulate broker hiccup on the ml queue
|
||||
return {"queues": out, "fetched_at": "x"}
|
||||
monkeypatch.setattr(activity_module, "_read_queues_sync", _partial)
|
||||
|
||||
resp = await client.get("/api/system/activity/queues")
|
||||
body = await resp.get_json()
|
||||
assert body["queues"]["ml"] is None
|
||||
assert body["queues"]["import"] == 0
|
||||
|
||||
|
||||
# --- /workers ------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workers_returns_per_worker_queue_membership(client, monkeypatch):
|
||||
def _fake():
|
||||
return {
|
||||
"workers": {
|
||||
"worker@host1": {"queues": ["import", "thumbnail"], "active_count": 2},
|
||||
"worker@host2": {"queues": ["ml"], "active_count": 0},
|
||||
},
|
||||
"fetched_at": "x",
|
||||
}
|
||||
monkeypatch.setattr(activity_module, "_read_workers_sync", _fake)
|
||||
|
||||
resp = await client.get("/api/system/activity/workers")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert "worker@host1" in body["workers"]
|
||||
assert "import" in body["workers"]["worker@host1"]["queues"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workers_cached_within_ttl(client, monkeypatch):
|
||||
call_count = {"n": 0}
|
||||
|
||||
def _counting():
|
||||
call_count["n"] += 1
|
||||
return {"workers": {}, "fetched_at": "x"}
|
||||
monkeypatch.setattr(activity_module, "_read_workers_sync", _counting)
|
||||
|
||||
await client.get("/api/system/activity/workers")
|
||||
await client.get("/api/system/activity/workers")
|
||||
assert call_count["n"] == 1
|
||||
|
||||
|
||||
# --- /runs ---------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def _seed_runs(db):
|
||||
"""Insert 5 task_run rows for paging tests."""
|
||||
now = datetime.now(UTC)
|
||||
for i in range(5):
|
||||
db.add(TaskRun(
|
||||
celery_task_id=f"tid-{i}",
|
||||
queue="import" if i % 2 == 0 else "ml",
|
||||
task_name=f"backend.app.tasks.fake.task_{i}",
|
||||
target_id=i,
|
||||
started_at=now - timedelta(seconds=10 - i),
|
||||
finished_at=now - timedelta(seconds=9 - i),
|
||||
duration_ms=1000,
|
||||
status="ok" if i < 3 else "error",
|
||||
error_type="ValueError" if i >= 3 else None,
|
||||
error_message="boom" if i >= 3 else None,
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_paginated_descending_by_id(client, _seed_runs):
|
||||
resp = await client.get("/api/system/activity/runs?limit=3")
|
||||
body = await resp.get_json()
|
||||
assert len(body["runs"]) == 3
|
||||
# Descending: latest id first.
|
||||
ids = [r["id"] for r in body["runs"]]
|
||||
assert ids == sorted(ids, reverse=True)
|
||||
assert body["next_cursor"] is not None # 5 total, asked for 3 → more
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_filter_by_queue(client, _seed_runs):
|
||||
resp = await client.get("/api/system/activity/runs?queue=ml")
|
||||
body = await resp.get_json()
|
||||
assert all(r["queue"] == "ml" for r in body["runs"])
|
||||
assert len(body["runs"]) == 2 # i=1, i=3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_filter_by_status(client, _seed_runs):
|
||||
resp = await client.get("/api/system/activity/runs?status=error")
|
||||
body = await resp.get_json()
|
||||
assert all(r["status"] == "error" for r in body["runs"])
|
||||
assert len(body["runs"]) == 2 # i=3, i=4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_keyset_cursor(client, _seed_runs):
|
||||
page1 = await (await client.get("/api/system/activity/runs?limit=2")).get_json()
|
||||
assert len(page1["runs"]) == 2
|
||||
assert page1["next_cursor"] is not None
|
||||
|
||||
page2 = await (await client.get(
|
||||
f"/api/system/activity/runs?limit=2&before_id={page1['next_cursor']}"
|
||||
)).get_json()
|
||||
assert len(page2["runs"]) == 2
|
||||
page1_ids = {r["id"] for r in page1["runs"]}
|
||||
page2_ids = {r["id"] for r in page2["runs"]}
|
||||
assert page1_ids.isdisjoint(page2_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_invalid_limit_400(client):
|
||||
resp = await client.get("/api/system/activity/runs?limit=not-a-number")
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# --- /failures -----------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def _seed_failures(db):
|
||||
"""Mix of ok / error / timeout rows; some old, some recent."""
|
||||
now = datetime.now(UTC)
|
||||
recent = [
|
||||
("error", "OperationalError"),
|
||||
("error", "OperationalError"),
|
||||
("error", "OperationalError"),
|
||||
("timeout", "TimeoutError"),
|
||||
("timeout", "TimeoutError"),
|
||||
("error", "OSError"),
|
||||
("ok", None), # success — should NOT appear in failures
|
||||
]
|
||||
for i, (status, etype) in enumerate(recent):
|
||||
db.add(TaskRun(
|
||||
celery_task_id=f"f-{i}",
|
||||
queue="ml",
|
||||
task_name="backend.app.tasks.fake.x",
|
||||
target_id=i,
|
||||
started_at=now - timedelta(seconds=120 - i),
|
||||
finished_at=now - timedelta(seconds=60 - i),
|
||||
duration_ms=1000,
|
||||
status=status,
|
||||
error_type=etype,
|
||||
error_message="boom" if status != "ok" else None,
|
||||
))
|
||||
# One ancient failure (>24h) that should NOT appear.
|
||||
db.add(TaskRun(
|
||||
celery_task_id="f-old",
|
||||
queue="ml",
|
||||
task_name="backend.app.tasks.fake.x",
|
||||
target_id=999,
|
||||
started_at=now - timedelta(hours=30),
|
||||
finished_at=now - timedelta(hours=29),
|
||||
duration_ms=1000,
|
||||
status="error",
|
||||
error_type="OldError",
|
||||
error_message="ancient",
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failures_returns_recent_errors_and_timeouts(client, _seed_failures):
|
||||
resp = await client.get("/api/system/activity/failures")
|
||||
body = await resp.get_json()
|
||||
statuses = {r["status"] for r in body["recent"]}
|
||||
assert statuses == {"error", "timeout"}
|
||||
# No ok rows.
|
||||
assert "ok" not in statuses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failures_count_by_type_groups_correctly(client, _seed_failures):
|
||||
resp = await client.get("/api/system/activity/failures")
|
||||
body = await resp.get_json()
|
||||
counts = body["count_by_type"]
|
||||
assert counts.get("OperationalError") == 3
|
||||
assert counts.get("TimeoutError") == 2
|
||||
assert counts.get("OSError") == 1
|
||||
assert "OldError" not in counts # outside 24h window
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failures_only_within_24h_window(client, _seed_failures):
|
||||
resp = await client.get("/api/system/activity/failures")
|
||||
body = await resp.get_json()
|
||||
ids = {r["error_type"] for r in body["recent"]}
|
||||
assert "OldError" not in ids
|
||||
@@ -0,0 +1,336 @@
|
||||
"""FC-3h: /api/system/backup/* endpoint integration tests."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app import create_app
|
||||
from backend.app.models import BackupRun
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def app():
|
||||
return create_app()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(app):
|
||||
async with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def _seed_runs(db):
|
||||
"""Insert 4 BackupRun rows for paging/filter tests."""
|
||||
now = datetime.now(UTC)
|
||||
for i in range(4):
|
||||
db.add(BackupRun(
|
||||
kind="db" if i % 2 == 0 else "images",
|
||||
status="ok",
|
||||
tag=None,
|
||||
triggered_by="manual",
|
||||
started_at=now - timedelta(seconds=10 - i),
|
||||
finished_at=now - timedelta(seconds=9 - i),
|
||||
sql_path="/tmp/fake.sql" if i % 2 == 0 else None,
|
||||
tar_path=None if i % 2 == 0 else "/tmp/fake.tar.zst",
|
||||
size_bytes=100,
|
||||
manifest={},
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
|
||||
# --- POST /db, /images ----------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_db_dispatches_and_returns_202(client, monkeypatch):
|
||||
dispatched = []
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.backup.backup_db_task.delay",
|
||||
lambda **kw: dispatched.append(kw),
|
||||
)
|
||||
resp = await client.post("/api/system/backup/db", json={})
|
||||
assert resp.status_code == 202
|
||||
body = await resp.get_json()
|
||||
assert body["status"] == "dispatched"
|
||||
assert dispatched == [{"tag": None, "triggered_by": "manual"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_db_persists_tag(client, monkeypatch):
|
||||
dispatched = []
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.backup.backup_db_task.delay",
|
||||
lambda **kw: dispatched.append(kw),
|
||||
)
|
||||
resp = await client.post(
|
||||
"/api/system/backup/db", json={"tag": "pre-cutover"},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
assert dispatched[0]["tag"] == "pre-cutover"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_db_rejects_tag_too_long(client):
|
||||
resp = await client.post(
|
||||
"/api/system/backup/db", json={"tag": "x" * 65},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "invalid_tag"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_images_dispatches(client, monkeypatch):
|
||||
dispatched = []
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.backup.backup_images_task.delay",
|
||||
lambda **kw: dispatched.append(kw),
|
||||
)
|
||||
resp = await client.post("/api/system/backup/images", json={})
|
||||
assert resp.status_code == 202
|
||||
|
||||
|
||||
# --- GET /runs -------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_paginated_descending(client, _seed_runs):
|
||||
resp = await client.get("/api/system/backup/runs?limit=2")
|
||||
body = await resp.get_json()
|
||||
assert len(body["runs"]) == 2
|
||||
ids = [r["id"] for r in body["runs"]]
|
||||
assert ids == sorted(ids, reverse=True)
|
||||
assert body["next_cursor"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_filter_by_kind_db(client, _seed_runs):
|
||||
resp = await client.get("/api/system/backup/runs?kind=db")
|
||||
body = await resp.get_json()
|
||||
assert all(r["kind"] == "db" for r in body["runs"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_filter_by_kind_images(client, _seed_runs):
|
||||
resp = await client.get("/api/system/backup/runs?kind=images")
|
||||
body = await resp.get_json()
|
||||
assert all(r["kind"] == "images" for r in body["runs"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_rejects_invalid_kind(client):
|
||||
resp = await client.get("/api/system/backup/runs?kind=bogus")
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runs_rejects_invalid_limit(client):
|
||||
resp = await client.get("/api/system/backup/runs?limit=not-int")
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# --- GET /runs/<id> --------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_run_returns_row(client, _seed_runs):
|
||||
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
resp = await client.get(f"/api/system/backup/runs/{rid}")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["id"] == rid
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_run_404(client):
|
||||
resp = await client.get("/api/system/backup/runs/999999")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# --- PATCH /runs/<id> ------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_tag_sets_value(client, _seed_runs, db_sync):
|
||||
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
|
||||
resp = await client.patch(
|
||||
f"/api/system/backup/runs/{rid}",
|
||||
json={"tag": "monthly"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
tag = db_sync.execute(
|
||||
select(BackupRun.tag).where(BackupRun.id == rid)
|
||||
).scalar_one()
|
||||
assert tag == "monthly"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_tag_null_clears(client, _seed_runs, db_sync):
|
||||
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
|
||||
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": "x"})
|
||||
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": None})
|
||||
|
||||
tag = db_sync.execute(
|
||||
select(BackupRun.tag).where(BackupRun.id == rid)
|
||||
).scalar_one()
|
||||
assert tag is None
|
||||
|
||||
|
||||
# --- POST /runs/<id>/restore ----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_wrong_confirm_400(client, _seed_runs):
|
||||
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/system/backup/runs/{rid}/restore",
|
||||
json={"confirm": "wrong"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "confirm_mismatch"
|
||||
assert body["expected"] == f"restore-db-{rid}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_correct_confirm_dispatches(client, _seed_runs, monkeypatch):
|
||||
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
|
||||
dispatched = []
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.backup.restore_db_task.delay",
|
||||
lambda **kw: dispatched.append(kw),
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/system/backup/runs/{rid}/restore",
|
||||
json={"confirm": f"restore-db-{rid}"},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
assert dispatched == [{"source_backup_run_id": rid}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_rejects_non_ok_status(client, db):
|
||||
# Seed an error-status row directly.
|
||||
db.add(BackupRun(
|
||||
kind="db", status="error", tag=None, triggered_by="manual",
|
||||
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
|
||||
sql_path="/tmp/fake.sql", manifest={},
|
||||
))
|
||||
await db.commit()
|
||||
rid_q = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
||||
rid = (await rid_q.get_json())["runs"][0]["id"]
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/system/backup/runs/{rid}/restore",
|
||||
json={"confirm": f"restore-db-{rid}"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "not_restorable"
|
||||
|
||||
|
||||
# --- DELETE /runs/<id> ----------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_wrong_confirm_400(client, _seed_runs):
|
||||
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
||||
rid = (await list_resp.get_json())["runs"][0]["id"]
|
||||
|
||||
resp = await client.delete(
|
||||
f"/api/system/backup/runs/{rid}",
|
||||
json={"confirm": "wrong"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_correct_confirm_204(client, _seed_runs, db_sync):
|
||||
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
||||
row = (await list_resp.get_json())["runs"][0]
|
||||
rid = row["id"]
|
||||
kind = row["kind"]
|
||||
|
||||
resp = await client.delete(
|
||||
f"/api/system/backup/runs/{rid}",
|
||||
json={"confirm": f"delete-{kind}-{rid}"},
|
||||
)
|
||||
assert resp.status_code == 204
|
||||
|
||||
surviving = db_sync.execute(
|
||||
select(BackupRun.id).where(BackupRun.id == rid)
|
||||
).scalar_one_or_none()
|
||||
assert surviving is None
|
||||
|
||||
|
||||
# --- /settings ------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_settings_returns_defaults(client):
|
||||
resp = await client.get("/api/system/backup/settings")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["backup_db_nightly_enabled"] is False
|
||||
assert body["backup_db_nightly_hour_utc"] == 3
|
||||
assert body["backup_db_keep_last_n"] == 14
|
||||
assert body["backup_images_keep_last_n"] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_settings_updates_values(client):
|
||||
resp = await client.patch(
|
||||
"/api/system/backup/settings",
|
||||
json={
|
||||
"backup_db_nightly_enabled": True,
|
||||
"backup_db_nightly_hour_utc": 5,
|
||||
"backup_db_keep_last_n": 30,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["backup_db_nightly_enabled"] is True
|
||||
assert body["backup_db_nightly_hour_utc"] == 5
|
||||
assert body["backup_db_keep_last_n"] == 30
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_settings_rejects_invalid_bool(client):
|
||||
resp = await client.patch(
|
||||
"/api/system/backup/settings",
|
||||
json={"backup_db_nightly_enabled": "yes"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_settings_rejects_hour_out_of_range(client):
|
||||
resp = await client.patch(
|
||||
"/api/system/backup/settings",
|
||||
json={"backup_db_nightly_hour_utc": 24},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_settings_rejects_keep_n_below_min(client):
|
||||
resp = await client.patch(
|
||||
"/api/system/backup/settings",
|
||||
json={"backup_db_keep_last_n": 0},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,200 @@
|
||||
"""FC-3h: backup_service unit tests.
|
||||
|
||||
Subprocess calls (pg_dump, tar, psql) are monkeypatched so tests run
|
||||
without external binaries. The real subprocess behavior is exercised
|
||||
implicitly via the Celery task tests in test_tasks_backup.py.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services import backup_service
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_subprocess(monkeypatch):
|
||||
"""Replace subprocess.run with a fake that writes a sentinel to
|
||||
the target path (for pg_dump's -f, for tar's -cf). Captures all
|
||||
calls in a list."""
|
||||
calls = []
|
||||
|
||||
class _FakeProc:
|
||||
returncode = 0
|
||||
stdout = b""
|
||||
stderr = b""
|
||||
|
||||
def _fake_run(cmd, **kwargs):
|
||||
calls.append(list(cmd))
|
||||
if cmd[0] == "pg_dump":
|
||||
i = cmd.index("-f")
|
||||
Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
|
||||
elif cmd[0] == "tar" and "-cf" in cmd:
|
||||
i = cmd.index("-cf")
|
||||
Path(cmd[i + 1]).write_bytes(b"fake tar payload")
|
||||
return _FakeProc()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", _fake_run)
|
||||
return calls
|
||||
|
||||
|
||||
# --- backup_db -------------------------------------------------------
|
||||
|
||||
|
||||
def test_backup_db_writes_sql_and_manifest(tmp_path, fake_subprocess):
|
||||
result = backup_service.backup_db(
|
||||
db_url="postgresql://test@x/db",
|
||||
images_root=tmp_path,
|
||||
tag="pre-cutover",
|
||||
triggered_by="manual",
|
||||
)
|
||||
sql = Path(result["sql_path"])
|
||||
manifest = Path(result["manifest_path"])
|
||||
assert sql.is_file() and sql.suffix == ".sql"
|
||||
assert manifest.is_file() and manifest.suffix == ".json"
|
||||
assert result["kind"] == "db"
|
||||
assert result["tar_path"] is None
|
||||
assert result["size_bytes"] == len(b"-- fake pg_dump\n")
|
||||
|
||||
parsed = json.loads(manifest.read_text())
|
||||
assert parsed["kind"] == "db"
|
||||
assert parsed["tag"] == "pre-cutover"
|
||||
assert parsed["triggered_by"] == "manual"
|
||||
assert parsed["artifact_path"] == str(sql)
|
||||
|
||||
|
||||
def test_backup_db_strips_sqlalchemy_psycopg_driver(tmp_path, fake_subprocess):
|
||||
backup_service.backup_db(
|
||||
db_url="postgresql+psycopg://u:p@h/d", images_root=tmp_path,
|
||||
)
|
||||
cmd = fake_subprocess[0]
|
||||
assert cmd[0] == "pg_dump"
|
||||
assert cmd[-1].startswith("postgresql://")
|
||||
assert "+psycopg" not in cmd[-1]
|
||||
|
||||
|
||||
def test_backup_db_strips_asyncpg_driver(tmp_path, fake_subprocess):
|
||||
backup_service.backup_db(
|
||||
db_url="postgresql+asyncpg://u:p@h/d", images_root=tmp_path,
|
||||
)
|
||||
assert "+asyncpg" not in fake_subprocess[0][-1]
|
||||
|
||||
|
||||
def test_backup_db_default_tag_is_none(tmp_path, fake_subprocess):
|
||||
result = backup_service.backup_db(
|
||||
db_url="postgresql://u@h/d", images_root=tmp_path,
|
||||
)
|
||||
parsed = json.loads(Path(result["manifest_path"]).read_text())
|
||||
assert parsed["tag"] is None
|
||||
|
||||
|
||||
# --- backup_images ---------------------------------------------------
|
||||
|
||||
|
||||
def test_backup_images_writes_tar_and_manifest(tmp_path, fake_subprocess):
|
||||
result = backup_service.backup_images(
|
||||
images_root=tmp_path, tag="monthly", triggered_by="manual",
|
||||
)
|
||||
tar = Path(result["tar_path"])
|
||||
assert tar.is_file() and tar.name.endswith(".tar.zst")
|
||||
assert result["sql_path"] is None
|
||||
assert result["size_bytes"] == len(b"fake tar payload")
|
||||
|
||||
|
||||
def test_backup_images_excludes_backups_and_quarantine(tmp_path, fake_subprocess):
|
||||
backup_service.backup_images(images_root=tmp_path)
|
||||
cmd = fake_subprocess[0]
|
||||
excludes = [arg for arg in cmd if arg.startswith("--exclude=")]
|
||||
assert any("_backups" in e for e in excludes)
|
||||
assert any("_quarantine" in e for e in excludes)
|
||||
|
||||
|
||||
# --- restore_db ------------------------------------------------------
|
||||
|
||||
|
||||
def test_restore_db_drops_schema_then_loads(tmp_path, fake_subprocess):
|
||||
sql_path = tmp_path / "fake.sql"
|
||||
sql_path.write_text("SELECT 1;")
|
||||
backup_service.restore_db(
|
||||
db_url="postgresql://u@h/d", sql_path=sql_path,
|
||||
)
|
||||
# Two psql calls: one with -c (DROP SCHEMA), one with -f (load).
|
||||
assert len(fake_subprocess) == 2
|
||||
assert "-c" in fake_subprocess[0]
|
||||
assert "DROP SCHEMA IF EXISTS public CASCADE" in fake_subprocess[0][-1]
|
||||
assert "-f" in fake_subprocess[1]
|
||||
assert str(sql_path) in fake_subprocess[1]
|
||||
|
||||
|
||||
# --- restore_images --------------------------------------------------
|
||||
|
||||
|
||||
def test_restore_images_untar_to_parent(tmp_path, fake_subprocess):
|
||||
tar_path = tmp_path / "fake.tar.zst"
|
||||
tar_path.write_bytes(b"fake")
|
||||
backup_service.restore_images(images_root=tmp_path, tar_path=tar_path)
|
||||
cmd = fake_subprocess[0]
|
||||
assert cmd[:3] == ["tar", "--zstd", "-xf"]
|
||||
assert str(tar_path) in cmd
|
||||
assert "-C" in cmd
|
||||
|
||||
|
||||
# --- unlink ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_unlink_removes_present_files_and_reports(tmp_path):
|
||||
sql = tmp_path / "x.sql"
|
||||
sql.write_bytes(b"x")
|
||||
tar = tmp_path / "x.tar.zst"
|
||||
tar.write_bytes(b"x")
|
||||
manifest = tmp_path / "x.json"
|
||||
manifest.write_text("{}")
|
||||
result = backup_service.unlink_artifact_files(
|
||||
sql_path=str(sql), tar_path=str(tar), manifest_path=str(manifest),
|
||||
)
|
||||
assert result == {"sql": True, "tar": True, "manifest": True}
|
||||
assert not sql.exists() and not tar.exists() and not manifest.exists()
|
||||
|
||||
|
||||
def test_unlink_missing_files_returns_true(tmp_path):
|
||||
"""missing_ok semantics: a non-existent file isn't an error."""
|
||||
result = backup_service.unlink_artifact_files(
|
||||
sql_path=str(tmp_path / "nope.sql"),
|
||||
tar_path=None, manifest_path=None,
|
||||
)
|
||||
assert result == {"sql": True}
|
||||
|
||||
|
||||
def test_unlink_skips_none_paths():
|
||||
"""A None path is skipped — not added to the result dict."""
|
||||
result = backup_service.unlink_artifact_files(
|
||||
sql_path=None, tar_path=None, manifest_path=None,
|
||||
)
|
||||
assert result == {}
|
||||
|
||||
|
||||
# --- helpers ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_libpq_url_strips_each_known_driver():
|
||||
assert backup_service._libpq_url(
|
||||
"postgresql+psycopg://u@h/d"
|
||||
) == "postgresql://u@h/d"
|
||||
assert backup_service._libpq_url(
|
||||
"postgresql+asyncpg://u@h/d"
|
||||
) == "postgresql://u@h/d"
|
||||
assert backup_service._libpq_url(
|
||||
"postgresql+psycopg2://u@h/d"
|
||||
) == "postgresql://u@h/d"
|
||||
# Plain URL passes through unchanged.
|
||||
assert backup_service._libpq_url(
|
||||
"postgresql://u@h/d"
|
||||
) == "postgresql://u@h/d"
|
||||
|
||||
|
||||
def test_backups_dir_created_on_first_use(tmp_path):
|
||||
d = backup_service._backups_dir(tmp_path)
|
||||
assert d.is_dir()
|
||||
assert d.name == "_backups"
|
||||
@@ -0,0 +1,335 @@
|
||||
"""FC-3k: cleanup_service unit tests.
|
||||
|
||||
Mutations go against real Postgres (db_sync fixture). File-system
|
||||
side effects use tmp_path. Assertions on mutated rows use COLUMN
|
||||
SELECTS per reference_async_coredml_test_assertions — never
|
||||
re-read ORM attributes after a service mutates and re-fetches.
|
||||
"""
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Tag, TagKind
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services import cleanup_service
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _make_image(db_sync, *, artist, path, sha256, size=1000, thumb=None):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id,
|
||||
path=path,
|
||||
sha256=sha256,
|
||||
size_bytes=size,
|
||||
mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
thumbnail_path=thumb,
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
return img
|
||||
|
||||
|
||||
def _make_artist(db_sync, *, slug="aria", name="Aria"):
|
||||
a = Artist(name=name, slug=slug)
|
||||
db_sync.add(a)
|
||||
db_sync.flush()
|
||||
return a
|
||||
|
||||
|
||||
def _make_tag(db_sync, *, name, kind=TagKind.general):
|
||||
t = Tag(name=name, kind=kind)
|
||||
db_sync.add(t)
|
||||
db_sync.flush()
|
||||
return t
|
||||
|
||||
|
||||
# --- project_artist_cascade -----------------------------------------
|
||||
|
||||
|
||||
def test_project_artist_cascade_returns_zeroes_for_empty_artist(db_sync):
|
||||
_make_artist(db_sync, slug="empty")
|
||||
db_sync.commit()
|
||||
result = cleanup_service.project_artist_cascade(db_sync, slug="empty")
|
||||
assert result["artist"]["slug"] == "empty"
|
||||
assert result["projected"] == {
|
||||
"images": 0,
|
||||
"sources": 0,
|
||||
"thumbs": 0,
|
||||
"import_tasks": 0,
|
||||
"bytes_on_disk": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_project_artist_cascade_counts_images_and_thumbs_and_bytes(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="counted")
|
||||
_make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "a.jpg"),
|
||||
sha256="a" * 64, size=1000, thumb=str(tmp_path / "a.thumb"),
|
||||
)
|
||||
_make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "b.jpg"),
|
||||
sha256="b" * 64, size=2500, thumb=None,
|
||||
)
|
||||
db_sync.commit()
|
||||
result = cleanup_service.project_artist_cascade(db_sync, slug="counted")
|
||||
assert result["projected"]["images"] == 2
|
||||
assert result["projected"]["thumbs"] == 1
|
||||
assert result["projected"]["bytes_on_disk"] == 3500
|
||||
|
||||
|
||||
def test_project_artist_cascade_raises_on_unknown_slug(db_sync):
|
||||
with pytest.raises(LookupError):
|
||||
cleanup_service.project_artist_cascade(db_sync, slug="nope")
|
||||
|
||||
|
||||
# --- project_bulk_image_delete --------------------------------------
|
||||
|
||||
|
||||
def test_project_bulk_image_delete_separates_found_from_missing(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="bd")
|
||||
i1 = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "1.jpg"),
|
||||
sha256="1" * 64, size=10, thumb="t1",
|
||||
)
|
||||
i2 = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "2.jpg"),
|
||||
sha256="2" * 64, size=20, thumb=None,
|
||||
)
|
||||
db_sync.commit()
|
||||
result = cleanup_service.project_bulk_image_delete(
|
||||
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
||||
)
|
||||
assert result["images_found"] == 2
|
||||
assert result["thumbs_to_unlink"] == 1
|
||||
assert result["bytes_on_disk"] == 30
|
||||
assert result["missing_ids"] == [9_999_999]
|
||||
|
||||
|
||||
def test_project_bulk_image_delete_empty_input(db_sync):
|
||||
result = cleanup_service.project_bulk_image_delete(db_sync, image_ids=[])
|
||||
assert result == {
|
||||
"images_found": 0,
|
||||
"thumbs_to_unlink": 0,
|
||||
"bytes_on_disk": 0,
|
||||
"missing_ids": [],
|
||||
}
|
||||
|
||||
|
||||
# --- count_tag_associations / find_unused_tags ----------------------
|
||||
|
||||
|
||||
def test_count_tag_associations_counts_image_tag_rows(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="ta")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="c" * 64,
|
||||
)
|
||||
tag = _make_tag(db_sync, name="cyberpunk")
|
||||
db_sync.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=tag.id,
|
||||
))
|
||||
db_sync.commit()
|
||||
assert cleanup_service.count_tag_associations(db_sync, tag_id=tag.id) == 1
|
||||
|
||||
|
||||
def test_find_unused_tags_returns_only_unreferenced(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="fu")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "y.jpg"), sha256="d" * 64,
|
||||
)
|
||||
used = _make_tag(db_sync, name="used")
|
||||
_make_tag(db_sync, name="aaa-unused")
|
||||
_make_tag(db_sync, name="zzz-unused")
|
||||
db_sync.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=used.id,
|
||||
))
|
||||
db_sync.commit()
|
||||
result = cleanup_service.find_unused_tags(db_sync)
|
||||
names = [t.name for t in result]
|
||||
assert "used" not in names
|
||||
assert "aaa-unused" in names
|
||||
assert "zzz-unused" in names
|
||||
# Sorted by name ascending.
|
||||
assert names.index("aaa-unused") < names.index("zzz-unused")
|
||||
|
||||
|
||||
# --- unlink_image_files ---------------------------------------------
|
||||
|
||||
|
||||
def test_unlink_image_files_removes_original_and_thumbnail(db_sync, tmp_path):
|
||||
original = tmp_path / "orig.jpg"
|
||||
original.write_bytes(b"x")
|
||||
custom_thumb = tmp_path / "custom.thumb"
|
||||
custom_thumb.write_bytes(b"x")
|
||||
conv_thumb = tmp_path / "thumbs" / "aaa" / ("a" * 64 + ".jpg")
|
||||
conv_thumb.parent.mkdir(parents=True)
|
||||
conv_thumb.write_bytes(b"x")
|
||||
|
||||
img = ImageRecord(
|
||||
artist_id=None, path=str(original), sha256="a" * 64,
|
||||
size_bytes=1, thumbnail_path=str(custom_thumb),
|
||||
)
|
||||
result = cleanup_service.unlink_image_files(img, tmp_path)
|
||||
assert result == {"original": True, "thumbnail": True}
|
||||
assert not original.exists()
|
||||
assert not custom_thumb.exists()
|
||||
assert not conv_thumb.exists()
|
||||
|
||||
|
||||
def test_unlink_image_files_missing_files_count_as_success(tmp_path):
|
||||
img = ImageRecord(
|
||||
artist_id=None, path=str(tmp_path / "nope.jpg"),
|
||||
sha256="b" * 64, size_bytes=1, thumbnail_path=None,
|
||||
)
|
||||
result = cleanup_service.unlink_image_files(img, tmp_path)
|
||||
assert result == {"original": True, "thumbnail": False}
|
||||
|
||||
|
||||
# --- delete_artist_cascade ------------------------------------------
|
||||
|
||||
|
||||
def test_delete_artist_cascade_removes_images_and_artist_row(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="cas")
|
||||
for i in range(3):
|
||||
f = tmp_path / f"img{i}.jpg"
|
||||
f.write_bytes(b"x")
|
||||
_make_image(
|
||||
db_sync, artist=a, path=str(f),
|
||||
sha256=f"{i:064x}", size=10,
|
||||
)
|
||||
db_sync.commit()
|
||||
artist_id = a.id
|
||||
|
||||
result = cleanup_service.delete_artist_cascade(
|
||||
db_sync, artist_id=artist_id, images_root=tmp_path,
|
||||
)
|
||||
assert result["artist"]["slug"] == "cas"
|
||||
assert result["summary"]["images_deleted"] == 3
|
||||
assert result["summary"]["files_deleted"] == 3
|
||||
|
||||
# Column-select assertions per reference_async_coredml_test_assertions.
|
||||
surviving_artist = db_sync.execute(
|
||||
select(func.count(Artist.id)).where(Artist.id == artist_id)
|
||||
).scalar_one()
|
||||
surviving_images = db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.artist_id == artist_id)
|
||||
).scalar_one()
|
||||
assert surviving_artist == 0
|
||||
assert surviving_images == 0
|
||||
|
||||
|
||||
def test_delete_artist_cascade_idempotent_on_missing(db_sync, tmp_path):
|
||||
result = cleanup_service.delete_artist_cascade(
|
||||
db_sync, artist_id=9_999_999, images_root=tmp_path,
|
||||
)
|
||||
assert result["artist"] is None
|
||||
assert result["summary"]["images_deleted"] == 0
|
||||
|
||||
|
||||
# --- delete_images --------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_images_removes_rows_and_files(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="di")
|
||||
f1 = tmp_path / "1.jpg"
|
||||
f1.write_bytes(b"x")
|
||||
f2 = tmp_path / "2.jpg"
|
||||
f2.write_bytes(b"x")
|
||||
i1 = _make_image(db_sync, artist=a, path=str(f1), sha256="1" * 64)
|
||||
i2 = _make_image(db_sync, artist=a, path=str(f2), sha256="2" * 64)
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.delete_images(
|
||||
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
||||
images_root=tmp_path,
|
||||
)
|
||||
assert result["images_deleted"] == 2
|
||||
assert result["files_deleted"] == 2
|
||||
assert result["missing_ids"] == [9_999_999]
|
||||
|
||||
surviving = db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.id.in_([i1.id, i2.id]))
|
||||
).scalar_one()
|
||||
assert surviving == 0
|
||||
|
||||
|
||||
# --- delete_tag -----------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_tag_cascades_associations(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="dt")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="d" * 64,
|
||||
)
|
||||
tag = _make_tag(db_sync, name="doomed")
|
||||
db_sync.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=tag.id,
|
||||
))
|
||||
db_sync.commit()
|
||||
tag_id = tag.id
|
||||
|
||||
result = cleanup_service.delete_tag(db_sync, tag_id=tag_id)
|
||||
assert result["deleted"]["id"] == tag_id
|
||||
assert result["associations_removed"] == 1
|
||||
|
||||
surviving_tag = db_sync.execute(
|
||||
select(func.count(Tag.id)).where(Tag.id == tag_id)
|
||||
).scalar_one()
|
||||
surviving_assoc = db_sync.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag).where(image_tag.c.tag_id == tag_id)
|
||||
).scalar_one()
|
||||
assert surviving_tag == 0
|
||||
assert surviving_assoc == 0
|
||||
|
||||
|
||||
def test_delete_tag_raises_on_unknown_id(db_sync):
|
||||
with pytest.raises(LookupError):
|
||||
cleanup_service.delete_tag(db_sync, tag_id=9_999_999)
|
||||
|
||||
|
||||
# --- prune_unused_tags ----------------------------------------------
|
||||
|
||||
|
||||
def test_prune_unused_tags_dry_run_returns_count_and_names(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="pu")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "z.jpg"), sha256="e" * 64,
|
||||
)
|
||||
used = _make_tag(db_sync, name="kept")
|
||||
_make_tag(db_sync, name="prune-me-1")
|
||||
_make_tag(db_sync, name="prune-me-2")
|
||||
db_sync.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=used.id,
|
||||
))
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.prune_unused_tags(db_sync, dry_run=True)
|
||||
assert result["count"] == 2
|
||||
assert "prune-me-1" in result["sample_names"]
|
||||
assert "prune-me-2" in result["sample_names"]
|
||||
# Nothing actually deleted.
|
||||
surviving = db_sync.execute(select(func.count(Tag.id))).scalar_one()
|
||||
assert surviving == 3
|
||||
|
||||
|
||||
def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="pu2")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "k.jpg"), sha256="f" * 64,
|
||||
)
|
||||
used = _make_tag(db_sync, name="kept")
|
||||
_make_tag(db_sync, name="bye")
|
||||
db_sync.execute(image_tag.insert().values(
|
||||
image_record_id=img.id, tag_id=used.id,
|
||||
))
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.prune_unused_tags(db_sync, dry_run=False)
|
||||
assert result["deleted"] == 1
|
||||
|
||||
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
|
||||
assert "kept" in surviving_names
|
||||
assert "bye" not in surviving_names
|
||||
+177
-1
@@ -30,9 +30,14 @@ def test_recover_interrupted_only_old(db_sync, monkeypatch):
|
||||
batch_id = _make_batch(db_sync)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# "Fresh" must sit comfortably under whatever STUCK_THRESHOLD_MINUTES
|
||||
# currently is (5 min as of 2026-05-24, tightened from 30); 30
|
||||
# seconds is well below any reasonable threshold. "Stale" stays at
|
||||
# 2 hours so the test remains valid if the threshold ever moves
|
||||
# back up.
|
||||
fresh = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/a.jpg", task_type="media",
|
||||
status="processing", started_at=now - timedelta(minutes=5),
|
||||
status="processing", started_at=now - timedelta(seconds=30),
|
||||
)
|
||||
stale = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/b.jpg", task_type="media",
|
||||
@@ -88,3 +93,174 @@ def test_cleanup_old_deletes_finished_old(db_sync):
|
||||
|
||||
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
|
||||
assert remaining == {"/import/b.jpg", "/import/c.jpg"}
|
||||
|
||||
|
||||
# --- FC-3i: task_run sweep + retention -----------------------------
|
||||
|
||||
|
||||
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
|
||||
error_type=None):
|
||||
from backend.app.models import TaskRun
|
||||
row = TaskRun(
|
||||
celery_task_id="x",
|
||||
queue="ml",
|
||||
task_name="backend.app.tasks.fake.t",
|
||||
target_id=1,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at,
|
||||
duration_ms=1000 if finished_at else None,
|
||||
status=status,
|
||||
error_type=error_type,
|
||||
error_message="x" if status in ("error", "timeout") else None,
|
||||
)
|
||||
db_sync.add(row)
|
||||
db_sync.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
def test_recover_stalled_task_runs_flips_old_running_to_error(db_sync):
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
stale_id = _make_task_run(
|
||||
db_sync, status="running", started_at=now - timedelta(minutes=10),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
recovered = recover_stalled_task_runs.apply().get()
|
||||
assert recovered == 1
|
||||
|
||||
db_sync.expire_all()
|
||||
status = db_sync.execute(
|
||||
select(TaskRun.status).where(TaskRun.id == stale_id)
|
||||
).scalar_one()
|
||||
error_type = db_sync.execute(
|
||||
select(TaskRun.error_type).where(TaskRun.id == stale_id)
|
||||
).scalar_one()
|
||||
assert status == "error"
|
||||
assert error_type == "RecoverySweep"
|
||||
|
||||
|
||||
def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
fresh_id = _make_task_run(
|
||||
db_sync, status="running", started_at=now - timedelta(seconds=30),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
recovered = recover_stalled_task_runs.apply().get()
|
||||
assert recovered == 0
|
||||
|
||||
db_sync.expire_all()
|
||||
status = db_sync.execute(
|
||||
select(TaskRun.status).where(TaskRun.id == fresh_id)
|
||||
).scalar_one()
|
||||
assert status == "running"
|
||||
|
||||
|
||||
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import prune_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
old_id = _make_task_run(
|
||||
db_sync, status="ok",
|
||||
started_at=now - timedelta(hours=30),
|
||||
finished_at=now - timedelta(hours=29),
|
||||
)
|
||||
recent_id = _make_task_run(
|
||||
db_sync, status="ok",
|
||||
started_at=now - timedelta(hours=2),
|
||||
finished_at=now - timedelta(hours=1),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
result = prune_task_runs.apply().get()
|
||||
assert result["ok_deleted"] == 1
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving_ids = set(db_sync.execute(
|
||||
select(TaskRun.id).where(TaskRun.id.in_([old_id, recent_id]))
|
||||
).scalars().all())
|
||||
assert surviving_ids == {recent_id}
|
||||
|
||||
|
||||
def test_prune_task_runs_deletes_failures_older_than_7d(db_sync):
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import prune_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
old_id = _make_task_run(
|
||||
db_sync, status="error", error_type="OldError",
|
||||
started_at=now - timedelta(days=10),
|
||||
finished_at=now - timedelta(days=9),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
result = prune_task_runs.apply().get()
|
||||
assert result["failures_deleted"] >= 1
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving = db_sync.execute(
|
||||
select(TaskRun.id).where(TaskRun.id == old_id)
|
||||
).scalar_one_or_none()
|
||||
assert surviving is None
|
||||
|
||||
|
||||
def test_prune_task_runs_keeps_recent_failures(db_sync):
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import prune_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
recent_id = _make_task_run(
|
||||
db_sync, status="error", error_type="RecentError",
|
||||
started_at=now - timedelta(days=6),
|
||||
finished_at=now - timedelta(days=5),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
prune_task_runs.apply().get()
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving = db_sync.execute(
|
||||
select(TaskRun.id).where(TaskRun.id == recent_id)
|
||||
).scalar_one_or_none()
|
||||
assert surviving == recent_id
|
||||
|
||||
|
||||
def test_prune_task_runs_never_deletes_running(db_sync):
|
||||
"""Even a 30-day-old running row stays — recovery sweep is the
|
||||
mechanism that flips them; prune doesn't touch in-flight."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import TaskRun
|
||||
from backend.app.tasks.maintenance import prune_task_runs
|
||||
|
||||
now = datetime.now(UTC)
|
||||
ancient_id = _make_task_run(
|
||||
db_sync, status="running",
|
||||
started_at=now - timedelta(days=30),
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
prune_task_runs.apply().get()
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving = db_sync.execute(
|
||||
select(TaskRun.id).where(TaskRun.id == ancient_id)
|
||||
).scalar_one_or_none()
|
||||
assert surviving == ancient_id
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
"""FC-5: backup + rollback service tests.
|
||||
|
||||
Uses tmp_path to avoid touching real /images/_backups/. Subprocess
|
||||
calls (pg_dump, tar) are monkeypatched — the real shell-out is exercised
|
||||
in operator-side smoke testing on the homelab, not in unit tests.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.app.services.migrators import backup as backup_mod
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_backup_writes_sql_and_tar(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls.append(cmd)
|
||||
# Touch the expected output file so existence checks pass downstream.
|
||||
if "pg_dump" in cmd[0]:
|
||||
outpath = tmp_path / "_backups" / f"fc_{kwargs['_test_ts']}.sql"
|
||||
outpath.write_text("-- fake pg_dump output")
|
||||
else:
|
||||
outpath = tmp_path / "_backups" / f"fc_{kwargs['_test_ts']}.tar.zst"
|
||||
outpath.write_bytes(b"\x28\xb5\x2f\xfd") # zstd magic
|
||||
from types import SimpleNamespace
|
||||
return SimpleNamespace(returncode=0, stdout=b"", stderr=b"")
|
||||
|
||||
monkeypatch.setattr(backup_mod, "_run_subprocess", fake_run)
|
||||
|
||||
result = backup_mod.create_backup(
|
||||
db_url="postgresql://x", images_root=tmp_path, tag="pre_migration",
|
||||
)
|
||||
assert "sql_path" in result
|
||||
assert "tar_path" in result
|
||||
assert result["tag"] == "pre_migration"
|
||||
assert any("pg_dump" in c[0] for c in calls)
|
||||
assert any(c[0] == "tar" for c in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_latest_backup_by_tag(monkeypatch, tmp_path):
|
||||
# Create two manifests with different tags.
|
||||
backups_dir = tmp_path / "_backups"
|
||||
backups_dir.mkdir()
|
||||
import json
|
||||
(backups_dir / "fc_20260101T000000Z.json").write_text(json.dumps({
|
||||
"backup_id": "20260101T000000Z", "tag": "manual",
|
||||
"created_at": "2026-01-01T00:00:00+00:00",
|
||||
"sql_path": "/x/a.sql", "tar_path": "/x/a.tar.zst",
|
||||
}))
|
||||
(backups_dir / "fc_20260202T000000Z.json").write_text(json.dumps({
|
||||
"backup_id": "20260202T000000Z", "tag": "pre_migration",
|
||||
"created_at": "2026-02-02T00:00:00+00:00",
|
||||
"sql_path": "/x/b.sql", "tar_path": "/x/b.tar.zst",
|
||||
}))
|
||||
|
||||
found = backup_mod.find_latest_backup(tmp_path, tag="pre_migration")
|
||||
assert found is not None
|
||||
assert found["backup_id"] == "20260202T000000Z"
|
||||
|
||||
missing = backup_mod.find_latest_backup(tmp_path, tag="nonexistent")
|
||||
assert missing is None
|
||||
+146
-4
@@ -2,9 +2,19 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
SeriesPage,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
image_tag,
|
||||
)
|
||||
from backend.app.services.migrators import tag_apply
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
@@ -21,14 +31,19 @@ async def _seed_image(db, sha: str, *, suffix="x"):
|
||||
return img
|
||||
|
||||
|
||||
def _write_manifest(tmp_path, *, artist_assignments=None, tag_associations=None, series_pages=None):
|
||||
def _write_manifest(
|
||||
tmp_path, *,
|
||||
artist_assignments=None, tag_associations=None,
|
||||
series_pages=None, image_posts=None, schema_version=2,
|
||||
):
|
||||
d = tmp_path / "_migration_state"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "ir_tag_manifest.json").write_text(json.dumps({
|
||||
"schema_version": 1,
|
||||
"schema_version": schema_version,
|
||||
"image_artist_assignments": artist_assignments or [],
|
||||
"image_tag_associations": tag_associations or [],
|
||||
"series_pages": series_pages or [],
|
||||
"image_posts": image_posts or [],
|
||||
}))
|
||||
|
||||
|
||||
@@ -154,3 +169,130 @@ async def test_dry_run_makes_no_changes(db, tmp_path):
|
||||
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
|
||||
)).all()
|
||||
assert len(rows) == 0
|
||||
|
||||
|
||||
# --- Phase 4: image_posts (schema v2) -----------------------------------
|
||||
|
||||
|
||||
_POST_ENTRY = {
|
||||
"platform": "patreon",
|
||||
"post_id": "10001",
|
||||
"artist": "Maewix",
|
||||
"title": "Test Post Title",
|
||||
"description": "<p>HTML description</p>",
|
||||
"source_url": "https://www.patreon.com/posts/test-10001",
|
||||
"attachment_count": 1,
|
||||
"published_at": "2026-01-15T12:00:00+00:00",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_creates_source_post_provenance(db, tmp_path):
|
||||
sha = "d" * 64
|
||||
await _seed_image(db, sha, suffix="d")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_inserted"] >= 1
|
||||
|
||||
# Async Core-DML assertions — column selects, not ORM attribute access.
|
||||
artist_count = (await db.execute(
|
||||
select(func.count(Artist.id)).where(Artist.slug == "maewix")
|
||||
)).scalar_one()
|
||||
assert artist_count == 1
|
||||
|
||||
source_count = (await db.execute(
|
||||
select(func.count(Source.id)).where(
|
||||
Source.platform == "patreon",
|
||||
Source.url == "https://www.patreon.com/maewix",
|
||||
)
|
||||
)).scalar_one()
|
||||
assert source_count == 1
|
||||
|
||||
post_count = (await db.execute(
|
||||
select(func.count(Post.id)).where(Post.external_post_id == "10001")
|
||||
)).scalar_one()
|
||||
assert post_count == 1
|
||||
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
.where(ImageProvenance.image_record_id == img_id)
|
||||
)).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_idempotent_on_rerun(db, tmp_path):
|
||||
sha = "e" * 64
|
||||
await _seed_image(db, sha, suffix="e")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
# Re-run: nothing should be newly inserted; all skipped.
|
||||
assert result2["counts"]["rows_inserted"] == 0
|
||||
assert result2["counts"]["rows_skipped"] >= 1
|
||||
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
)).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_unknown_sha_unmatched(db, tmp_path):
|
||||
# No image seeded; sha doesn't exist.
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": ["f" * 64]},
|
||||
])
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
unmatched_kinds = [u["kind"] for u in result["unmatched"]]
|
||||
assert "post" in unmatched_kinds
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_unknown_platform_skipped(db, tmp_path):
|
||||
sha = "1" * 64
|
||||
await _seed_image(db, sha, suffix="1")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "platform": "tumblr", "image_sha256s": [sha]},
|
||||
])
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_skipped"] >= 1
|
||||
|
||||
# No Post / Source / ImageProvenance should have been created.
|
||||
src_count = (await db.execute(
|
||||
select(func.count(Source.id)).where(Source.platform == "tumblr")
|
||||
)).scalar_one()
|
||||
assert src_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_dry_run_makes_no_writes(db, tmp_path):
|
||||
sha = "2" * 64
|
||||
await _seed_image(db, sha, suffix="2")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
)).scalar_one()
|
||||
assert prov_count == 0
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""FC-3i: Celery signal → task_run row tests.
|
||||
|
||||
Uses task_always_eager so signals fire synchronously in-process during
|
||||
the test. Scoped autouse fixture keeps the eager mode confined to this
|
||||
file (other tests may break with eager mode enabled globally).
|
||||
|
||||
Test assertions on signal-mutated rows use COLUMN SELECTS / db_sync.
|
||||
scalar (per reference_async_coredml_test_assertions). ORM attribute
|
||||
access on a row a signal handler just mutated risks MissingGreenlet
|
||||
across async/sync boundaries.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.celery_app import celery
|
||||
from backend.app.models import TaskRun
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _eager_celery(monkeypatch):
|
||||
"""task_always_eager so signals fire in-process during the test."""
|
||||
monkeypatch.setattr(celery.conf, "task_always_eager", True)
|
||||
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prerun_inserts_running_row(db_sync):
|
||||
"""Fire a no-op task; assert task_run row inserted with running status."""
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
generate_thumbnail.delay(999_999) # nonexistent image_id → no-op
|
||||
|
||||
rows = db_sync.execute(
|
||||
select(TaskRun.queue, TaskRun.target_id, TaskRun.task_name)
|
||||
.where(TaskRun.target_id == 999_999)
|
||||
).all()
|
||||
assert len(rows) == 1
|
||||
queue, target_id, task_name = rows[0]
|
||||
assert queue == "thumbnail"
|
||||
assert target_id == 999_999
|
||||
assert task_name.endswith(".generate_thumbnail")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_postrun_marks_ok_with_duration(db_sync):
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
generate_thumbnail.delay(999_998)
|
||||
|
||||
row = db_sync.execute(
|
||||
select(
|
||||
TaskRun.status, TaskRun.finished_at, TaskRun.duration_ms,
|
||||
).where(TaskRun.target_id == 999_998)
|
||||
).one()
|
||||
status, finished_at, duration_ms = row
|
||||
assert status == "ok"
|
||||
assert finished_at is not None
|
||||
assert duration_ms is not None and duration_ms >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_records_error_type_and_message(db_sync, monkeypatch):
|
||||
"""Force a tracked task to raise; assert error fields populated."""
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
|
||||
def _explode(*a, **k):
|
||||
raise ValueError("synthetic failure")
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.thumbnail.generate_thumbnail.run",
|
||||
_explode,
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
generate_thumbnail.delay(999_997).get(propagate=True)
|
||||
|
||||
row = db_sync.execute(
|
||||
select(
|
||||
TaskRun.status, TaskRun.error_type, TaskRun.error_message,
|
||||
TaskRun.finished_at,
|
||||
).where(TaskRun.target_id == 999_997)
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).one()
|
||||
status, error_type, error_message, finished_at = row
|
||||
assert status == "error"
|
||||
assert error_type == "ValueError"
|
||||
assert "synthetic failure" in (error_message or "")
|
||||
assert finished_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_time_limit_records_timeout_status(db_sync, monkeypatch):
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
|
||||
def _timeout(*a, **k):
|
||||
raise SoftTimeLimitExceeded("simulated soft limit")
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.thumbnail.generate_thumbnail.run",
|
||||
_timeout,
|
||||
)
|
||||
with pytest.raises(SoftTimeLimitExceeded):
|
||||
generate_thumbnail.delay(999_996).get(propagate=True)
|
||||
|
||||
status = db_sync.execute(
|
||||
select(TaskRun.status).where(TaskRun.target_id == 999_996)
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).scalar_one()
|
||||
assert status == "timeout"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untracked_celery_internal_tasks_skip(db_sync):
|
||||
"""celery.chord_unlock is in the untracked list; firing it should
|
||||
NOT insert a task_run row."""
|
||||
before = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
|
||||
# Send a chord_unlock-style task via apply (no real task to dispatch).
|
||||
# The signal handler checks task.name against _UNTRACKED_TASK_NAMES;
|
||||
# if name is in the list, no row is inserted. We verify by firing
|
||||
# a regular task and asserting only ONE row appears, not two.
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
generate_thumbnail.delay(999_995)
|
||||
after = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
|
||||
assert after - before == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_args_summary_truncated_to_255(db_sync, monkeypatch):
|
||||
"""Task with a very large repr(args); assert args_summary <= 255."""
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
|
||||
# Force a giant arg by patching repr(args) target — actually just
|
||||
# call with a large int that produces a long repr.
|
||||
big_id = int("9" * 200) # 200-char repr
|
||||
# The signal handler reads repr(args), so this naturally produces
|
||||
# a long args_summary that should get truncated to 255.
|
||||
try:
|
||||
generate_thumbnail.delay(big_id).get(timeout=5, propagate=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
args_summary = db_sync.execute(
|
||||
select(TaskRun.args_summary)
|
||||
.where(TaskRun.task_name.endswith(".generate_thumbnail"))
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).scalar_one()
|
||||
assert args_summary is None or len(args_summary) <= 255
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_message_truncated_to_2000(db_sync, monkeypatch):
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
|
||||
huge_message = "X" * 5000
|
||||
|
||||
def _raise_huge(*a, **k):
|
||||
raise RuntimeError(huge_message)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.thumbnail.generate_thumbnail.run", _raise_huge,
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
generate_thumbnail.delay(999_994).get(propagate=True)
|
||||
|
||||
error_message = db_sync.execute(
|
||||
select(TaskRun.error_message).where(TaskRun.target_id == 999_994)
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).scalar_one()
|
||||
assert error_message is not None
|
||||
assert len(error_message) <= 2000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signal_handler_failure_does_not_break_task(db_sync, monkeypatch):
|
||||
"""Monkeypatch sync_session_factory in the signals module to raise;
|
||||
assert the task still returns its result and the worker doesn't
|
||||
propagate the signal-handler exception."""
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
|
||||
def _broken_factory():
|
||||
raise RuntimeError("db down")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.celery_signals.sync_session_factory", _broken_factory,
|
||||
)
|
||||
|
||||
# Task should still execute (no-op on missing image) without raising.
|
||||
result = generate_thumbnail.delay(999_993).get(timeout=5)
|
||||
assert result is not None # task ran; signal handler errors swallowed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args,expected", [
|
||||
((42,), 42),
|
||||
((42, "ignored"), 42),
|
||||
(("123",), 123),
|
||||
(("not-int",), None),
|
||||
((), None),
|
||||
(None, None),
|
||||
])
|
||||
def test_target_id_extracted_from_first_int_arg(args, expected):
|
||||
"""Unit test the helper directly; no DB needed."""
|
||||
from backend.app.celery_signals import _target_id_from_args
|
||||
assert _target_id_from_args(args) == expected
|
||||
|
||||
|
||||
def test_finalize_sets_retry_status_with_retry_count(db_sync):
|
||||
"""Direct unit-shape test of _finalize's retry path. We can't
|
||||
reliably trigger Celery's task_retry signal under eager mode with
|
||||
monkeypatched .run (the bind=True 'self' injection breaks), so
|
||||
test the finalize helper directly — that's the code the retry
|
||||
signal handler delegates to anyway."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from backend.app.celery_signals import _finalize
|
||||
from backend.app.models import TaskRun
|
||||
|
||||
# Seed a 'running' row that _finalize can target.
|
||||
row = TaskRun(
|
||||
celery_task_id="retry-test-tid",
|
||||
queue="ml",
|
||||
task_name="backend.app.tasks.fake.t",
|
||||
target_id=999_992,
|
||||
started_at=datetime.now(UTC),
|
||||
status="running",
|
||||
)
|
||||
db_sync.add(row)
|
||||
db_sync.commit()
|
||||
|
||||
_finalize(
|
||||
"retry-test-tid", status="retry",
|
||||
error_type="RuntimeError",
|
||||
error_message="first attempt",
|
||||
retry_count=1,
|
||||
)
|
||||
|
||||
# Column-select assertions — _finalize mutated the row via its own
|
||||
# session; re-read fresh state through db_sync.
|
||||
db_sync.expire_all()
|
||||
status, retry_count, error_type = db_sync.execute(
|
||||
select(TaskRun.status, TaskRun.retry_count, TaskRun.error_type)
|
||||
.where(TaskRun.celery_task_id == "retry-test-tid")
|
||||
).one()
|
||||
assert status == "retry"
|
||||
assert retry_count == 1
|
||||
assert error_type == "RuntimeError"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_hostname_recorded(db_sync):
|
||||
"""Sender's hostname should land in worker_hostname."""
|
||||
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||
generate_thumbnail.delay(999_991)
|
||||
|
||||
hostname = db_sync.execute(
|
||||
select(TaskRun.worker_hostname).where(TaskRun.target_id == 999_991)
|
||||
).scalar_one()
|
||||
# Under eager mode hostname may be None or "celery@local"; just
|
||||
# assert the column is touched (either string or None — never throws).
|
||||
assert hostname is None or isinstance(hostname, str)
|
||||
@@ -0,0 +1,158 @@
|
||||
"""FC-3k: admin Celery task integration tests.
|
||||
|
||||
task_always_eager + signal handlers from FC-3i populate task_run.
|
||||
We assert the wrapper passes args through correctly and that
|
||||
task_run lifecycle status flips as expected.
|
||||
"""
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
import backend.app.tasks.admin # noqa: F401 — register tasks
|
||||
from backend.app.celery_app import celery
|
||||
from backend.app.models import Artist, ImageRecord, TaskRun
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _eager_celery(monkeypatch):
|
||||
monkeypatch.setattr(celery.conf, "task_always_eager", True)
|
||||
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_images_root(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.app.tasks.admin.IMAGES_ROOT", tmp_path)
|
||||
|
||||
|
||||
# --- registration ----------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_artist_cascade_task_registered():
|
||||
assert (
|
||||
"backend.app.tasks.admin.delete_artist_cascade_task"
|
||||
in celery.tasks
|
||||
)
|
||||
|
||||
|
||||
def test_bulk_delete_images_task_registered():
|
||||
assert (
|
||||
"backend.app.tasks.admin.bulk_delete_images_task"
|
||||
in celery.tasks
|
||||
)
|
||||
|
||||
|
||||
# --- delete_artist_cascade_task -------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_artist_cascade_task_removes_artist_and_records_ok(
|
||||
db_sync, tmp_path,
|
||||
):
|
||||
from backend.app.tasks.admin import delete_artist_cascade_task
|
||||
|
||||
a = Artist(name="Doomed", slug="doomed")
|
||||
db_sync.add(a)
|
||||
db_sync.flush()
|
||||
for i in range(2):
|
||||
f = tmp_path / f"d{i}.jpg"
|
||||
f.write_bytes(b"x")
|
||||
db_sync.add(ImageRecord(
|
||||
artist_id=a.id, path=str(f),
|
||||
sha256=f"{i:064x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
))
|
||||
db_sync.commit()
|
||||
artist_id = a.id
|
||||
|
||||
result = delete_artist_cascade_task.delay(artist_id=artist_id).get()
|
||||
assert result["summary"]["images_deleted"] == 2
|
||||
|
||||
surviving = db_sync.execute(
|
||||
select(func.count(Artist.id)).where(Artist.id == artist_id)
|
||||
).scalar_one()
|
||||
assert surviving == 0
|
||||
|
||||
# FC-3i task_run lifecycle: task should have an 'ok' row.
|
||||
status = db_sync.execute(
|
||||
select(TaskRun.status)
|
||||
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).scalar_one()
|
||||
assert status == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_artist_cascade_task_records_failure(
|
||||
db_sync, monkeypatch,
|
||||
):
|
||||
from backend.app.tasks.admin import delete_artist_cascade_task
|
||||
|
||||
def _boom(*a, **kw):
|
||||
raise RuntimeError("synthetic cascade fail")
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.cleanup_service.delete_artist_cascade", _boom,
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
delete_artist_cascade_task.delay(artist_id=1).get()
|
||||
|
||||
row = db_sync.execute(
|
||||
select(TaskRun.status, TaskRun.error_message)
|
||||
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
|
||||
.order_by(TaskRun.id.desc()).limit(1)
|
||||
).one()
|
||||
assert row.status == "error"
|
||||
assert "synthetic" in (row.error_message or "")
|
||||
|
||||
|
||||
# --- bulk_delete_images_task ----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_images_task_removes_listed_images(
|
||||
db_sync, tmp_path,
|
||||
):
|
||||
from backend.app.tasks.admin import bulk_delete_images_task
|
||||
|
||||
a = Artist(name="B", slug="b")
|
||||
db_sync.add(a)
|
||||
db_sync.flush()
|
||||
ids = []
|
||||
for i in range(3):
|
||||
f = tmp_path / f"b{i}.jpg"
|
||||
f.write_bytes(b"x")
|
||||
img = ImageRecord(
|
||||
artist_id=a.id, path=str(f),
|
||||
sha256=f"a{i:063x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
ids.append(img.id)
|
||||
db_sync.commit()
|
||||
|
||||
result = bulk_delete_images_task.delay(image_ids=ids).get()
|
||||
assert result["images_deleted"] == 3
|
||||
|
||||
surviving = db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.id.in_(ids))
|
||||
).scalar_one()
|
||||
assert surviving == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_images_task_idempotent_on_empty_list(db_sync):
|
||||
from backend.app.tasks.admin import bulk_delete_images_task
|
||||
result = bulk_delete_images_task.delay(image_ids=[]).get()
|
||||
assert result["images_deleted"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_images_task_reports_missing_ids(db_sync):
|
||||
from backend.app.tasks.admin import bulk_delete_images_task
|
||||
result = bulk_delete_images_task.delay(
|
||||
image_ids=[9_999_998, 9_999_999],
|
||||
).get()
|
||||
assert result["images_deleted"] == 0
|
||||
assert sorted(result["missing_ids"]) == [9_999_998, 9_999_999]
|
||||
@@ -0,0 +1,281 @@
|
||||
"""FC-3h: backup/restore Celery task integration tests.
|
||||
|
||||
Uses task_always_eager for synchronous in-test execution.
|
||||
Subprocess + IMAGES_ROOT are monkeypatched so tests don't touch
|
||||
real /images or shell out to pg_dump.
|
||||
"""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.celery_app import celery
|
||||
from backend.app.models import BackupRun, ImportSettings
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _eager_celery(monkeypatch):
|
||||
monkeypatch.setattr(celery.conf, "task_always_eager", True)
|
||||
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_subprocess_and_images_root(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.backup_service._DB_SUBPROCESS_TIMEOUT_S", 5,
|
||||
)
|
||||
|
||||
class _FakeProc:
|
||||
returncode = 0
|
||||
stdout = b""
|
||||
stderr = b""
|
||||
|
||||
def _fake_run(cmd, **kwargs):
|
||||
if cmd[0] == "pg_dump":
|
||||
i = cmd.index("-f")
|
||||
Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
|
||||
elif cmd[0] == "tar" and "-cf" in cmd:
|
||||
i = cmd.index("-cf")
|
||||
Path(cmd[i + 1]).write_bytes(b"fake tar payload")
|
||||
return _FakeProc()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", _fake_run)
|
||||
|
||||
|
||||
def _seed_backup(db_sync, *, kind, status, started_at, tag=None,
|
||||
finished_at=None):
|
||||
row = BackupRun(
|
||||
kind=kind, status=status, tag=tag,
|
||||
triggered_by="manual", started_at=started_at,
|
||||
finished_at=finished_at or (started_at + timedelta(seconds=10)),
|
||||
sql_path="/tmp/fake.sql" if kind == "db" else None,
|
||||
tar_path="/tmp/fake.tar.zst" if kind == "images" else None,
|
||||
manifest={},
|
||||
)
|
||||
db_sync.add(row)
|
||||
db_sync.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
# --- backup_db_task --------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_db_task_creates_backup_run_row_status_ok(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_task
|
||||
result = backup_db_task.delay(tag=None, triggered_by="manual").get()
|
||||
run_id = result["backup_run_id"]
|
||||
row = db_sync.execute(
|
||||
select(
|
||||
BackupRun.kind, BackupRun.status, BackupRun.sql_path,
|
||||
BackupRun.size_bytes, BackupRun.finished_at, BackupRun.error,
|
||||
).where(BackupRun.id == run_id)
|
||||
).one()
|
||||
assert row.kind == "db"
|
||||
assert row.status == "ok"
|
||||
assert row.sql_path and row.sql_path.endswith(".sql")
|
||||
assert row.size_bytes is not None and row.size_bytes > 0
|
||||
assert row.finished_at is not None
|
||||
assert row.error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_db_task_records_failure_on_subprocess_error(db_sync, monkeypatch):
|
||||
from backend.app.tasks.backup import backup_db_task
|
||||
|
||||
def _boom(*a, **kw):
|
||||
raise RuntimeError("synthetic pg_dump fail")
|
||||
monkeypatch.setattr("subprocess.run", _boom)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
backup_db_task.delay().get()
|
||||
|
||||
row = db_sync.execute(
|
||||
select(BackupRun.status, BackupRun.error)
|
||||
.where(BackupRun.kind == "db")
|
||||
.order_by(BackupRun.id.desc()).limit(1)
|
||||
).one()
|
||||
assert row.status == "error"
|
||||
assert "synthetic" in (row.error or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_db_task_persists_tag(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_task
|
||||
result = backup_db_task.delay(tag="pre-cutover").get()
|
||||
tag = db_sync.execute(
|
||||
select(BackupRun.tag).where(BackupRun.id == result["backup_run_id"])
|
||||
).scalar_one()
|
||||
assert tag == "pre-cutover"
|
||||
|
||||
|
||||
# --- backup_images_task ---------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_images_task_creates_backup_run(db_sync):
|
||||
from backend.app.tasks.backup import backup_images_task
|
||||
result = backup_images_task.delay().get()
|
||||
row = db_sync.execute(
|
||||
select(BackupRun.kind, BackupRun.status, BackupRun.tar_path)
|
||||
.where(BackupRun.id == result["backup_run_id"])
|
||||
).one()
|
||||
assert row.kind == "images"
|
||||
assert row.status == "ok"
|
||||
assert row.tar_path and row.tar_path.endswith(".tar.zst")
|
||||
|
||||
|
||||
# --- restore_db_task ------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_db_task_creates_restoring_marker_then_restored(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_task, restore_db_task
|
||||
src = backup_db_task.delay().get()
|
||||
src_id = src["backup_run_id"]
|
||||
|
||||
restore_db_task.delay(source_backup_run_id=src_id).get()
|
||||
|
||||
rows = db_sync.execute(
|
||||
select(BackupRun.status, BackupRun.triggered_by)
|
||||
.where(BackupRun.restored_from_id == src_id)
|
||||
).all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status == "restored"
|
||||
assert rows[0].triggered_by == "restore"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_db_task_rejects_non_db_source(db_sync):
|
||||
from backend.app.tasks.backup import restore_db_task
|
||||
|
||||
now = datetime.now(UTC)
|
||||
img_id = _seed_backup(db_sync, kind="images", status="ok", started_at=now)
|
||||
db_sync.commit()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
restore_db_task.delay(source_backup_run_id=img_id).get()
|
||||
|
||||
|
||||
# --- prune_backups --------------------------------------------------
|
||||
|
||||
|
||||
def test_prune_backups_keeps_last_n_untagged_per_kind(db_sync):
|
||||
from backend.app.tasks.backup import prune_backups
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_keep_last_n = 2
|
||||
s.backup_images_keep_last_n = 1
|
||||
db_sync.commit()
|
||||
|
||||
now = datetime.now(UTC)
|
||||
for i in range(4):
|
||||
_seed_backup(db_sync, kind="db", status="ok",
|
||||
started_at=now - timedelta(hours=i))
|
||||
for i in range(3):
|
||||
_seed_backup(db_sync, kind="images", status="ok",
|
||||
started_at=now - timedelta(hours=i))
|
||||
db_sync.commit()
|
||||
|
||||
result = prune_backups.apply().get()
|
||||
assert result["db_deleted"] == 2
|
||||
assert result["images_deleted"] == 2
|
||||
|
||||
|
||||
def test_prune_backups_protects_tagged_rows(db_sync):
|
||||
from backend.app.tasks.backup import prune_backups
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_keep_last_n = 1
|
||||
db_sync.commit()
|
||||
|
||||
now = datetime.now(UTC)
|
||||
_seed_backup(db_sync, kind="db", status="ok", started_at=now)
|
||||
tagged_id = _seed_backup(
|
||||
db_sync, kind="db", status="ok",
|
||||
started_at=now - timedelta(days=30), tag="forever",
|
||||
)
|
||||
_seed_backup(db_sync, kind="db", status="ok",
|
||||
started_at=now - timedelta(days=1))
|
||||
db_sync.commit()
|
||||
|
||||
prune_backups.apply().get()
|
||||
|
||||
surviving_tag = db_sync.execute(
|
||||
select(BackupRun.tag).where(BackupRun.id == tagged_id)
|
||||
).scalar_one_or_none()
|
||||
assert surviving_tag == "forever"
|
||||
|
||||
|
||||
def test_prune_backups_never_deletes_running_or_restoring(db_sync):
|
||||
from backend.app.tasks.backup import prune_backups
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_keep_last_n = 0
|
||||
db_sync.commit()
|
||||
|
||||
ancient = datetime.now(UTC) - timedelta(days=30)
|
||||
running_id = _seed_backup(db_sync, kind="db", status="running", started_at=ancient)
|
||||
restoring_id = _seed_backup(db_sync, kind="db", status="restoring", started_at=ancient)
|
||||
db_sync.commit()
|
||||
|
||||
prune_backups.apply().get()
|
||||
|
||||
statuses = db_sync.execute(
|
||||
select(BackupRun.status).where(BackupRun.id.in_([running_id, restoring_id]))
|
||||
).scalars().all()
|
||||
assert set(statuses) == {"running", "restoring"}
|
||||
|
||||
|
||||
# --- backup_db_nightly ----------------------------------------------
|
||||
|
||||
|
||||
def test_nightly_skips_when_disabled(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_nightly
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_nightly_enabled = False
|
||||
db_sync.commit()
|
||||
|
||||
result = backup_db_nightly.apply().get()
|
||||
assert "skipped" in result and "disabled" in result["skipped"]
|
||||
|
||||
|
||||
def test_nightly_skips_when_hour_mismatch(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_nightly
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_nightly_enabled = True
|
||||
s.backup_db_nightly_hour_utc = (datetime.now(UTC).hour + 12) % 24
|
||||
db_sync.commit()
|
||||
|
||||
result = backup_db_nightly.apply().get()
|
||||
assert "skipped" in result and "hour=" in result["skipped"]
|
||||
|
||||
|
||||
def test_nightly_dispatches_when_enabled_at_configured_hour(db_sync):
|
||||
from backend.app.tasks.backup import backup_db_nightly
|
||||
|
||||
s = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
s.backup_db_nightly_enabled = True
|
||||
s.backup_db_nightly_hour_utc = datetime.now(UTC).hour
|
||||
db_sync.commit()
|
||||
|
||||
result = backup_db_nightly.apply().get()
|
||||
assert "dispatched" in result
|
||||
Reference in New Issue
Block a user