Compare commits

...

20 Commits

Author SHA1 Message Date
bvandeusen d1c0b82a22 Merge pull request 'v26.05.24.3: FC-3i System Activity dashboard + migration backup-gate retired + modal Escape' (#9) from dev into main 2026-05-24 21:47:53 -04:00
bvandeusen 37fcc74954 fix(fc3i): single-line celery.signals import + INT32 bounds on target_id + dict.fromkeys + rewrite retry test as direct _finalize unit
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:22:42 -04:00
bvandeusen 6a532c1497 fc3i(ui): Settings → Activity tab + Overview summary card
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:10:14 -04:00
bvandeusen 368613068a fc3i(ui): SystemActivityTab.vue — queues + failures + all-activity panes
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:09:20 -04:00
bvandeusen ddbb84d8aa fc3i(ui): SystemActivitySummary.vue — Overview-tab quick summary card (5s poll)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:08:30 -04:00
bvandeusen 9b252948f9 fc3i(ui): QueuesTable.vue — shared queue/worker/recent table (compact + detailed)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:57 -04:00
bvandeusen 89d0cb2124 fc3i(ui): systemActivity Pinia store — queues, workers, runs, failures
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:28 -04:00
bvandeusen 36611cbe00 fc3i: tests for recover_stalled_task_runs + prune_task_runs
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:01 -04:00
bvandeusen 48ef22445a fc3i: integration tests for /api/system/activity/* endpoints
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:06:20 -04:00
bvandeusen e523d0ac94 fc3i: integration tests for Celery signal → task_run lifecycle
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:05:32 -04:00
bvandeusen 541e2bfe6a fc3i: /api/system/activity blueprint — queues, workers, runs, failures
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:04:34 -04:00
bvandeusen 7782672a51 fc3i: recover_stalled_task_runs + prune_task_runs maintenance tasks + Beat entries
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:03:33 -04:00
bvandeusen d12b51f6b7 fc3i: Celery signal handlers populate task_run on every task lifecycle event
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:01:47 -04:00
bvandeusen 0cda46fcdb fc3i: alembic 0016 — task_run table with single + composite indexes
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:00:46 -04:00
bvandeusen 79fee98db4 fc3i: TaskRun model — per-Celery-task lifecycle audit row
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:00:14 -04:00
bvandeusen b1d68929c5 fix(tests): drop test_post_apply_without_backup_rejected (gate retired in 5535677)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 20:15:10 -04:00
bvandeusen 553567738e fix: drop migration backup-gate (FC-3h supersedes) + modal Escape via document-level listener so video focus can't swallow it
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 14:38:33 -04:00
bvandeusen 5526b8dc78 Merge pull request 'v26.05.24.2: IR Post/Provenance restore + modal artist fallback' (#8) from dev into main 2026-05-24 14:30:06 -04:00
bvandeusen c9a3f12847 fix(lint): tag_apply.py — one blank line between imports and module-level comment (ruff I001, third bounce on this rule)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 14:22:05 -04:00
bvandeusen 538c1591e8 fc-3g-ext: IR Post/Provenance restore (tag_apply phase 4) + modal artist fallback
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 14:08:18 -04:00
24 changed files with 2300 additions and 44 deletions
+86
View File
@@ -0,0 +1,86 @@
"""fc3i: task_run table
Revision ID: 0016
Revises: 0015
Create Date: 2026-05-24
Additive only. New table records every Celery task attempt via signal
handlers (backend.app.celery_signals). Status is plain String(16) not
Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard-
fail at INSERT, String columns extend cleanly).
Composite indexes anticipate the three dashboard panes:
- (queue, started_at desc) — per-lane recent activity
- (status, started_at desc) — recent failures pane
- (task_name, started_at desc) — drill-down by task
Indexed columns get individual indexes via `index=True` on the model;
the composites below cover the multi-column lookups.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0016"
down_revision: Union[str, None] = "0015"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"task_run",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("celery_task_id", sa.String(length=64), nullable=False),
sa.Column("queue", sa.String(length=32), nullable=False),
sa.Column("task_name", sa.String(length=128), nullable=False),
sa.Column("target_id", sa.Integer(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=True),
sa.Column(
"status", sa.String(length=16), nullable=False,
server_default="running",
),
sa.Column("error_type", sa.String(length=128), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("retry_count", sa.Integer(), nullable=True),
sa.Column("worker_hostname", sa.String(length=128), nullable=True),
sa.Column("args_summary", sa.String(length=255), nullable=True),
)
# Single-column indexes (matches Mapped[...].index=True on model).
op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"])
op.create_index("ix_task_run_queue", "task_run", ["queue"])
op.create_index("ix_task_run_task_name", "task_run", ["task_name"])
op.create_index("ix_task_run_started_at", "task_run", ["started_at"])
op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"])
op.create_index("ix_task_run_status", "task_run", ["status"])
# Composite indexes for dashboard query patterns.
op.create_index(
"ix_task_run_queue_started",
"task_run", ["queue", sa.text("started_at DESC")],
)
op.create_index(
"ix_task_run_status_started",
"task_run", ["status", sa.text("started_at DESC")],
)
op.create_index(
"ix_task_run_name_started",
"task_run", ["task_name", sa.text("started_at DESC")],
)
def downgrade() -> None:
op.drop_index("ix_task_run_name_started", table_name="task_run")
op.drop_index("ix_task_run_status_started", table_name="task_run")
op.drop_index("ix_task_run_queue_started", table_name="task_run")
op.drop_index("ix_task_run_status", table_name="task_run")
op.drop_index("ix_task_run_finished_at", table_name="task_run")
op.drop_index("ix_task_run_started_at", table_name="task_run")
op.drop_index("ix_task_run_task_name", table_name="task_run")
op.drop_index("ix_task_run_queue", table_name="task_run")
op.drop_index("ix_task_run_celery_task_id", table_name="task_run")
op.drop_table("task_run")
+2
View File
@@ -33,6 +33,7 @@ def all_blueprints() -> list[Blueprint]:
from .showcase import showcase_bp from .showcase import showcase_bp
from .sources import sources_bp from .sources import sources_bp
from .suggestions import suggestions_bp from .suggestions import suggestions_bp
from .system_activity import system_activity_bp
from .tags import tags_bp from .tags import tags_bp
return [ return [
api_bp, api_bp,
@@ -44,6 +45,7 @@ def all_blueprints() -> list[Blueprint]:
artists_bp, artists_bp,
showcase_bp, showcase_bp,
settings_bp, settings_bp,
system_activity_bp,
import_admin_bp, import_admin_bp,
migrate_bp, migrate_bp,
suggestions_bp, suggestions_bp,
+6 -1
View File
@@ -24,7 +24,12 @@ _VALID_KINDS = frozenset({
"ml_queue", "verify", "rollback", "cleanup", "ml_queue", "verify", "rollback", "cleanup",
}) })
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"}) _INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"}) # Backup gate retired 2026-05-24 — operator-flagged the speculative-safety
# requirement was actively blocking the UI ingest path (backup itself is
# unreliable on large NFS-backed image libraries) and FC-3h will rewrite
# the backup surface as a first-class feature with its own scheduling
# + recovery. Leaving the constant for historical grep.
_APPLY_KINDS: frozenset[str] = frozenset()
def _bad(error: str, *, status: int = 400, **extra): def _bad(error: str, *, status: int = 400, **extra):
+209
View File
@@ -0,0 +1,209 @@
"""FC-3i: system activity dashboard endpoints.
Read-only. Combines Redis-broker queue depths (LLEN per queue),
Celery worker introspection (celery inspect), and the task_run DB
history into the surfaces the SystemActivityTab UI consumes.
All filesystem/sync-client work goes through asyncio.to_thread per
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
"""
from __future__ import annotations
import asyncio
import time
from datetime import UTC, datetime, timedelta
from quart import Blueprint, jsonify, request
from sqlalchemy import desc, func, select
from ..config import get_config
from ..extensions import get_session
from ..models import TaskRun
system_activity_bp = Blueprint(
"system_activity", __name__, url_prefix="/api/system/activity",
)
# Canonical queue order — must match celery_app.task_routes. UI renders
# in this order; queues with no LLEN response show as null rather than
# absent.
_QUEUE_NAMES = (
"default", "import", "thumbnail", "ml",
"download", "scan", "maintenance",
)
# Cache module-level so all requests share the cache between polls.
# Tests can reset via direct dict mutation if needed.
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
_QUEUE_CACHE_TTL = 2.0
_WORKER_CACHE_TTL = 5.0
def _read_queues_sync() -> dict:
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
asyncio.to_thread. Per-queue try/except returns None on failure so
one bad queue doesn't break the whole response."""
import redis # local import; only this endpoint needs it
cfg = get_config()
client = redis.Redis.from_url(cfg.celery_broker_url)
out: dict = {}
for name in _QUEUE_NAMES:
try:
out[name] = int(client.llen(name))
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
out[name] = None
return {
"queues": out,
"fetched_at": datetime.now(UTC).isoformat(),
}
def _read_workers_sync() -> dict:
"""celery inspect active_queues + active. Returns per-worker info."""
from ..celery_app import celery as celery_app
insp = celery_app.control.inspect(timeout=2.0)
active_queues = insp.active_queues() or {}
active_tasks = insp.active() or {}
workers: dict = {}
for hostname, queues in active_queues.items():
workers[hostname] = {
"queues": sorted({q["name"] for q in queues}),
"active_count": len(active_tasks.get(hostname, [])),
}
return {
"workers": workers,
"fetched_at": datetime.now(UTC).isoformat(),
}
@system_activity_bp.route("/queues", methods=["GET"])
async def get_queues():
"""Per-queue Redis LLEN. Cached 2s.
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
"""
now = time.time()
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
_QUEUE_CACHE["ts"] = now
return jsonify(_QUEUE_CACHE["data"])
@system_activity_bp.route("/workers", methods=["GET"])
async def get_workers():
"""Live celery inspect. Cached 5s.
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
"""
now = time.time()
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
_WORKER_CACHE["ts"] = now
return jsonify(_WORKER_CACHE["data"])
@system_activity_bp.route("/runs", methods=["GET"])
async def list_runs():
"""Paginated task_run history. Query params:
queue=<name> filter to one queue
status=<status> filter to one status (running/ok/error/timeout/retry)
limit=<int> default 50, max 200
before_id=<int> cursor for keyset pagination
Response: {runs: [...], next_cursor: id|null}
"""
try:
limit = min(int(request.args.get("limit", "50")), 200)
except ValueError:
return jsonify({"error": "invalid_limit"}), 400
if limit < 1:
return jsonify({"error": "invalid_limit"}), 400
queue = request.args.get("queue")
status = request.args.get("status")
before_id_raw = request.args.get("before_id")
before_id = int(before_id_raw) if before_id_raw else None
async with get_session() as session:
stmt = select(TaskRun).order_by(desc(TaskRun.id))
if queue:
stmt = stmt.where(TaskRun.queue == queue)
if status:
stmt = stmt.where(TaskRun.status == status)
if before_id is not None:
stmt = stmt.where(TaskRun.id < before_id)
stmt = stmt.limit(limit + 1)
rows = (await session.execute(stmt)).scalars().all()
has_more = len(rows) > limit
rows = rows[:limit]
return jsonify({
"runs": [_row_to_dict(r) for r in rows],
"next_cursor": rows[-1].id if has_more and rows else None,
})
@system_activity_bp.route("/failures", methods=["GET"])
async def list_failures():
"""Recent failures across all lanes (24h window).
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
"""
try:
limit = min(int(request.args.get("limit", "50")), 200)
except ValueError:
return jsonify({"error": "invalid_limit"}), 400
since = datetime.now(UTC) - timedelta(hours=24)
async with get_session() as session:
recent_stmt = (
select(TaskRun)
.where(TaskRun.status.in_(["error", "timeout"]))
.where(TaskRun.finished_at >= since)
.order_by(desc(TaskRun.finished_at))
.limit(limit)
)
recent = (await session.execute(recent_stmt)).scalars().all()
count_stmt = (
select(TaskRun.error_type, func.count(TaskRun.id))
.where(TaskRun.status.in_(["error", "timeout"]))
.where(TaskRun.finished_at >= since)
.group_by(TaskRun.error_type)
.order_by(desc(func.count(TaskRun.id)))
)
counts = (await session.execute(count_stmt)).all()
return jsonify({
"recent": [_row_to_dict(r) for r in recent],
"count_by_type": {
(row[0] or "Unknown"): row[1]
for row in counts
},
"since": since.isoformat(),
})
def _row_to_dict(r: TaskRun) -> dict:
return {
"id": r.id,
"queue": r.queue,
"task_name": r.task_name,
"target_id": r.target_id,
"celery_task_id": r.celery_task_id,
"started_at": r.started_at.isoformat() if r.started_at else None,
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
"duration_ms": r.duration_ms,
"status": r.status,
"error_type": r.error_type,
"error_message": r.error_message,
"retry_count": r.retry_count,
"worker_hostname": r.worker_hostname,
"args_summary": r.args_summary,
}
+11
View File
@@ -81,9 +81,20 @@ def make_celery() -> Celery:
"task": "backend.app.tasks.maintenance.cleanup_old_download_events", "task": "backend.app.tasks.maintenance.cleanup_old_download_events",
"schedule": 86400.0, # daily "schedule": 86400.0, # daily
}, },
"recover-stalled-task-runs": {
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
},
"prune-task-runs": {
"task": "backend.app.tasks.maintenance.prune_task_runs",
"schedule": 86400.0, # daily
},
}, },
timezone="UTC", timezone="UTC",
) )
# FC-3i: register task_run signal handlers (side-effect import).
from . import celery_signals # noqa: F401
return app return app
+210
View File
@@ -0,0 +1,210 @@
"""FC-3i: task_run lifecycle via Celery signals.
Subscribes to task_prerun / task_postrun / task_failure / task_retry
and persists one task_run row per task attempt. Drop-in for every
existing and future Celery task — no per-task instrumentation.
Signal handlers run inside the worker process (sync context); DB
writes go through the existing shared sync engine
(backend.app.tasks._sync_engine.sync_session_factory) — one engine
per worker process, not per-task, so we don't blow Postgres
max_connections under load (the reason FC-3g shared-engine fix
existed).
Failure-mode discipline (operator-pressed point): every handler is
wrapped in try/except that swallows + logs. If the DB is down or the
handler has a bug, the real task still runs — the dashboard goes
dark for that interval. Monitoring NEVER breaks the thing it's
monitoring.
"""
import logging
from datetime import UTC, datetime
from celery.exceptions import SoftTimeLimitExceeded
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
from .models import TaskRun
from .tasks._sync_engine import sync_session_factory
log = logging.getLogger(__name__)
# Celery-internal tasks that would generate dashboard noise without
# operational value. Conservative list; extend only when a specific
# task proves noisy.
_UNTRACKED_TASK_NAMES = frozenset({
"celery.chord_unlock",
"celery.backend_cleanup",
"celery.chunks",
})
_MAX_ERROR_MESSAGE_LEN = 2000
_MAX_ARGS_SUMMARY_LEN = 255
_MAX_WORKER_HOSTNAME_LEN = 128
# PostgreSQL Integer is signed 32-bit. Tasks called with a first-arg
# int outside this range (e.g. an absurdly large mock value, or a
# string-of-digits coercible to int but bigger than 2^31-1) would crash
# the INSERT with NumericValueOutOfRange. Bound the recorded value to
# the column's range; values outside become None (target_id is
# nullable, so this is safe).
_INT32_MAX = 2_147_483_647
_INT32_MIN = -2_147_483_648
def _queue_for(task) -> str:
"""Reverse the task→queue routing from celery_app.task_routes.
Keep in sync if task_routes is reordered."""
name = getattr(task, "name", "") or ""
if name.startswith("backend.app.tasks.import_file."):
return "import"
if name.startswith("backend.app.tasks.ml."):
return "ml"
if name.startswith("backend.app.tasks.thumbnail."):
return "thumbnail"
if name.startswith("backend.app.tasks.download."):
return "download"
if name.startswith("backend.app.tasks.scan."):
return "scan"
if name.startswith((
"backend.app.tasks.maintenance.",
"backend.app.tasks.migration.",
)):
return "maintenance"
return "default"
def _target_id_from_args(args) -> int | None:
"""Best-effort: if the first positional arg parses as int AND fits
in the column's signed-32-bit range, record it as target_id
(image_id, source_id, etc.). Never raises."""
if not args:
return None
try:
value = int(args[0])
except (TypeError, ValueError):
return None
if value < _INT32_MIN or value > _INT32_MAX:
return None
return value
def _truncate(s, limit: int) -> str | None:
if s is None:
return None
text = str(s)
return text if len(text) <= limit else text[:limit]
def _is_tracked(task_name: str | None) -> bool:
return bool(task_name) and task_name not in _UNTRACKED_TASK_NAMES
@task_prerun.connect
def _on_prerun(sender=None, task_id=None, task=None, args=None,
kwargs=None, **_):
if not _is_tracked(getattr(task, "name", None)):
return
try:
Session = sync_session_factory()
with Session() as session:
session.add(TaskRun(
celery_task_id=task_id or "",
queue=_queue_for(task),
task_name=task.name,
target_id=_target_id_from_args(args),
started_at=datetime.now(UTC),
status="running",
args_summary=_truncate(repr(args), _MAX_ARGS_SUMMARY_LEN),
worker_hostname=_truncate(
getattr(sender, "hostname", None),
_MAX_WORKER_HOSTNAME_LEN,
),
))
session.commit()
except Exception: # noqa: BLE001 — never break the worker
log.exception("task_run prerun insert failed (task=%s)",
getattr(task, "name", "?"))
def _finalize(task_id: str, *, status: str,
error_type: str | None = None,
error_message: str | None = None,
retry_count: int | None = None) -> None:
"""Shared write path for postrun/failure/retry. Picks the most-
recent task_run row for this celery_task_id that's still 'running'
(retries reuse the same celery_task_id; each new attempt's prerun
inserts a fresh row, so finalize targets the latest running row)."""
try:
from sqlalchemy import select
Session = sync_session_factory()
now = datetime.now(UTC)
with Session() as session:
row = session.execute(
select(TaskRun)
.where(TaskRun.celery_task_id == task_id)
.where(TaskRun.status == "running")
.order_by(TaskRun.id.desc())
.limit(1)
).scalar_one_or_none()
if row is None:
return # no prerun row (untracked, insert failed, or already finalized)
row.finished_at = now
row.duration_ms = int(
(now - row.started_at).total_seconds() * 1000
)
row.status = status
if error_type is not None:
row.error_type = _truncate(error_type, 128)
if error_message is not None:
row.error_message = _truncate(error_message, _MAX_ERROR_MESSAGE_LEN)
if retry_count is not None:
row.retry_count = retry_count
session.commit()
except Exception: # noqa: BLE001
log.exception("task_run finalize failed (task_id=%s)", task_id)
@task_postrun.connect
def _on_postrun(sender=None, task_id=None, task=None, args=None,
kwargs=None, retval=None, state=None, **_):
if not _is_tracked(getattr(task, "name", None)):
return
# state is one of SUCCESS/FAILURE/RETRY/etc. Only handle SUCCESS;
# task_failure handles FAILURE explicitly (with the exception).
if state != "SUCCESS":
return
_finalize(task_id, status="ok")
@task_failure.connect
def _on_failure(sender=None, task_id=None, exception=None,
args=None, kwargs=None, einfo=None, **_):
if not _is_tracked(getattr(sender, "name", None)):
return
status = ("timeout"
if isinstance(exception, SoftTimeLimitExceeded)
else "error")
_finalize(
task_id, status=status,
error_type=type(exception).__name__ if exception else "Unknown",
error_message=str(exception) if exception else None,
)
@task_retry.connect
def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
if not _is_tracked(getattr(sender, "name", None)):
return
task_id = getattr(request, "id", None)
if not task_id:
return
# Mark current attempt's row as 'retry' (terminal for this row).
# The next attempt's task_prerun inserts a fresh row.
_finalize(
task_id, status="retry",
error_type=type(reason).__name__ if reason else "Retry",
error_message=str(reason) if reason else None,
retry_count=getattr(request, "retries", 0),
)
+2
View File
@@ -21,6 +21,7 @@ from .tag_alias import TagAlias
from .tag_allowlist import TagAllowlist from .tag_allowlist import TagAllowlist
from .tag_reference_embedding import TagReferenceEmbedding from .tag_reference_embedding import TagReferenceEmbedding
from .tag_suggestion_rejection import TagSuggestionRejection from .tag_suggestion_rejection import TagSuggestionRejection
from .task_run import TaskRun
__all__ = [ __all__ = [
"Base", "Base",
@@ -46,4 +47,5 @@ __all__ = [
"TagAllowlist", "TagAllowlist",
"TagReferenceEmbedding", "TagReferenceEmbedding",
"TagSuggestionRejection", "TagSuggestionRejection",
"TaskRun",
] ]
+48
View File
@@ -0,0 +1,48 @@
"""FC-3i: task_run — per-Celery-task lifecycle audit row.
One row inserted by the task_prerun signal at task start, updated by
task_postrun / task_failure / task_retry. The shape supports the
SystemActivity dashboard's three panes: per-queue queue+worker summary,
recent failures (24h, grouped by error_type), and full paginated
activity history.
Retention: ok rows pruned after 24h, error/timeout after 7d (see
backend.app.tasks.maintenance.prune_task_runs).
Recovery: rows stuck in 'running' for >5 min flipped to 'error' by
backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
"""
from datetime import datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class TaskRun(Base):
__tablename__ = "task_run"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
celery_task_id: Mapped[str] = mapped_column(
String(64), nullable=False, index=True,
)
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True,
)
finished_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True,
)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
status: Mapped[str] = mapped_column(
String(16), nullable=False, default="running", index=True,
)
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
retry_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
worker_hostname: Mapped[str | None] = mapped_column(String(128), nullable=True)
args_summary: Mapped[str | None] = mapped_column(String(255), nullable=True)
+13
View File
@@ -237,6 +237,15 @@ class GalleryService:
) )
tags = (await self.session.execute(tag_stmt)).scalars().all() tags = (await self.session.execute(tag_stmt)).scalars().all()
neighbors = await self._neighbors(record) neighbors = await self._neighbors(record)
# Direct artist FK — used by the modal's ProvenancePanel as a
# fallback when ImageProvenance is empty (i.e., filesystem-
# imported images without a post-track provenance row). The
# source of truth for richer post-level data is still
# ImageProvenance/Post; this is just the "we at least know who
# made it" line.
artist = None
if record.artist_id is not None:
artist = await self.session.get(Artist, record.artist_id)
return { return {
"id": record.id, "id": record.id,
"path": record.path, "path": record.path,
@@ -249,6 +258,10 @@ class GalleryService:
"created_at": record.created_at.isoformat(), "created_at": record.created_at.isoformat(),
"thumbnail_url": thumbnail_url(record.sha256, record.mime), "thumbnail_url": thumbnail_url(record.sha256, record.mime),
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}", "image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
"artist": (
{"id": artist.id, "name": artist.name, "slug": artist.slug}
if artist is not None else None
),
"tags": [ "tags": [
{ {
"id": t.id, "id": t.id,
+7 -2
View File
@@ -70,7 +70,7 @@ async def migrate_async(
""" """
if data.get("source_app") != "imagerepo": if data.get("source_app") != "imagerepo":
raise ValueError("export source_app must be 'imagerepo'") raise ValueError("export source_app must be 'imagerepo'")
if data.get("schema_version") != 1: if data.get("schema_version") not in (1, 2):
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
counts = _zero_counts() counts = _zero_counts()
@@ -126,15 +126,20 @@ async def migrate_async(
await db.commit() await db.commit()
# Phase 2: write the per-image manifest for tag_apply.py to consume later. # Phase 2: write the per-image manifest for tag_apply.py to consume later.
# schema_version 2 (added 2026-05-24) carries `image_posts` for
# Post + Source + ImageProvenance restore; schema 1 manifests
# without it stay valid (tag_apply treats the missing field as []).
manifest = { manifest = {
"schema_version": 1, "schema_version": data.get("schema_version", 1),
"image_artist_assignments": data.get("image_artist_assignments", []), "image_artist_assignments": data.get("image_artist_assignments", []),
"image_tag_associations": data.get("image_tag_associations", []), "image_tag_associations": data.get("image_tag_associations", []),
"series_pages": data.get("series_pages", []), "series_pages": data.get("series_pages", []),
"image_posts": data.get("image_posts", []),
} }
counts["rows_processed"] += len(manifest["image_artist_assignments"]) counts["rows_processed"] += len(manifest["image_artist_assignments"])
counts["rows_processed"] += len(manifest["image_tag_associations"]) counts["rows_processed"] += len(manifest["image_tag_associations"])
counts["rows_processed"] += len(manifest["series_pages"]) counts["rows_processed"] += len(manifest["series_pages"])
counts["rows_processed"] += len(manifest["image_posts"])
if not dry_run: if not dry_run:
manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
+166 -1
View File
@@ -7,6 +7,7 @@ FC's filesystem scan over the mounted IR images dir).
- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug). - image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug).
- image_tag_associations → image_tag insert (idempotent). - image_tag_associations → image_tag insert (idempotent).
- series_pages → series_page insert (idempotent on image_id unique). - series_pages → series_page insert (idempotent on image_id unique).
- image_posts (schema v2) → Source + Post + ImageProvenance restore.
Unmatched sha256s are logged into the result's `unmatched` list so the Unmatched sha256s are logged into the result's `unmatched` list so the
Celery task can drop them into MigrationRun.metadata for the operator Celery task can drop them into MigrationRun.metadata for the operator
@@ -15,15 +16,118 @@ to inspect.
from __future__ import annotations from __future__ import annotations
import json import json
from datetime import datetime
from pathlib import Path from pathlib import Path
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag from ...models import (
Artist,
ImageProvenance,
ImageRecord,
Post,
SeriesPage,
Source,
Tag,
TagKind,
image_tag,
)
from ...utils.slug import slugify from ...utils.slug import slugify
from .ir_ingest import manifest_path from .ir_ingest import manifest_path
# Per-platform artist-profile URL — used as Source.url when restoring
# IR PostMetadata into FC. Keep this table in sync with
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS and
# extension/lib/platforms.js.
_PLATFORM_PROFILE_URL = {
"patreon": "https://www.patreon.com/{slug}",
"subscribestar": "https://www.subscribestar.com/{slug}",
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
}
def _profile_url(platform: str, artist_slug: str) -> str | None:
fmt = _PLATFORM_PROFILE_URL.get(platform)
return fmt.format(slug=artist_slug) if fmt else None
async def _find_or_create_source(
db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool,
) -> int | None:
existing = (await db.execute(
select(Source.id).where(
Source.artist_id == artist_id,
Source.platform == platform,
Source.url == url,
)
)).scalar_one_or_none()
if existing is not None:
return existing
if dry_run:
return None
s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False)
db.add(s)
await db.flush()
return s.id
async def _find_or_create_post(
db: AsyncSession, *,
source_id: int, external_post_id: str,
title: str | None, description: str | None, post_url: str | None,
post_date_iso: str | None, attachment_count: int, dry_run: bool,
) -> int | None:
existing = (await db.execute(
select(Post.id).where(
Post.source_id == source_id,
Post.external_post_id == external_post_id,
)
)).scalar_one_or_none()
if existing is not None:
return existing
if dry_run:
return None
post_date = None
if post_date_iso:
post_date = datetime.fromisoformat(post_date_iso)
p = Post(
source_id=source_id,
external_post_id=external_post_id,
post_title=title,
description=description,
post_url=post_url,
post_date=post_date,
attachment_count=attachment_count,
raw_metadata={"migrated_from": "imagerepo"},
)
db.add(p)
await db.flush()
return p.id
async def _ensure_provenance(
db: AsyncSession, *,
image_id: int, post_id: int, source_id: int, dry_run: bool,
) -> bool:
"""Returns True if a new ImageProvenance row was inserted."""
existing = (await db.execute(
select(ImageProvenance.id).where(
ImageProvenance.image_record_id == image_id,
ImageProvenance.post_id == post_id,
ImageProvenance.source_id == source_id,
)
)).scalar_one_or_none()
if existing is not None:
return False
if dry_run:
return True
db.add(ImageProvenance(
image_record_id=image_id, post_id=post_id, source_id=source_id,
))
await db.flush()
return True
def _zero_counts() -> dict: def _zero_counts() -> dict:
return { return {
@@ -167,6 +271,67 @@ async def apply_async(
)) ))
counts["rows_inserted"] += 1 counts["rows_inserted"] += 1
# 4. Image posts (schema v2) → Source + Post + ImageProvenance.
# Restores IR PostMetadata as FC's downloader-track provenance,
# so the modal's ProvenancePanel surfaces title/description/
# source URL/publish date the same way it does for live
# gallery-dl downloads.
for entry in manifest.get("image_posts", []):
counts["rows_processed"] += 1
platform = entry.get("platform")
artist_name = entry.get("artist")
if not platform or not artist_name:
counts["rows_skipped"] += 1
continue
aid = await _ensure_artist_id(db, artist_name, dry_run)
if aid is None:
counts["rows_skipped"] += 1
continue
url = _profile_url(platform, slugify(artist_name))
if url is None:
counts["rows_skipped"] += 1
continue
source_id = await _find_or_create_source(
db, artist_id=aid, platform=platform, url=url, dry_run=dry_run,
)
if source_id is None:
counts["rows_skipped"] += 1
continue
post_id = await _find_or_create_post(
db, source_id=source_id,
external_post_id=entry.get("post_id") or "",
title=entry.get("title"),
description=entry.get("description"),
post_url=entry.get("source_url"),
post_date_iso=entry.get("published_at"),
attachment_count=entry.get("attachment_count") or 0,
dry_run=dry_run,
)
if post_id is None:
counts["rows_skipped"] += 1
continue
for sha in entry.get("image_sha256s", []):
img_id = await _sha_to_image_id(db, sha)
if img_id is None:
unmatched.append({
"kind": "post", "sha256": sha,
"post_id": entry.get("post_id"),
})
continue
inserted = await _ensure_provenance(
db, image_id=img_id, post_id=post_id,
source_id=source_id, dry_run=dry_run,
)
if inserted:
counts["rows_inserted"] += 1
else:
counts["rows_skipped"] += 1
if not dry_run: if not dry_run:
await db.commit() await db.commit()
return {"counts": counts, "unmatched": unmatched} return {"counts": counts, "unmatched": unmatched}
+68 -1
View File
@@ -9,7 +9,7 @@ from PIL import Image
from sqlalchemy import delete, select, update from sqlalchemy import delete, select, update
from ..celery_app import celery from ..celery_app import celery
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
from ..utils.phash import compute_phash from ..utils.phash import compute_phash
from ._sync_engine import sync_session_factory as _sync_session_factory from ._sync_engine import sync_session_factory as _sync_session_factory
@@ -20,6 +20,8 @@ OLD_TASK_DAYS = 7
PHASH_PAGE = 500 PHASH_PAGE = 500
VERIFY_PAGE = 200 VERIFY_PAGE = 200
FFPROBE_TIMEOUT_SECONDS = 10 FFPROBE_TIMEOUT_SECONDS = 10
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks") @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
@@ -80,6 +82,71 @@ def cleanup_old_tasks() -> int:
return result.rowcount or 0 return result.rowcount or 0
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
def recover_stalled_task_runs() -> int:
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
to 'error'. FC-3i.
A row gets stuck when the worker dies without emitting
task_postrun / task_failure (e.g. OOM, container restart between
signals, signal handler raised+logged). Shares the 5-min threshold
with recover_interrupted_tasks for consistency.
"""
SessionLocal = _sync_session_factory()
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
with SessionLocal() as session:
result = session.execute(
update(TaskRun)
.where(TaskRun.status == "running")
.where(TaskRun.started_at < cutoff)
.values(
status="error",
error_type="RecoverySweep",
error_message=(
f"no completion signal received within "
f"{STUCK_THRESHOLD_MINUTES} min"
),
finished_at=datetime.now(UTC),
)
)
session.commit()
return result.rowcount or 0
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
def prune_task_runs() -> dict:
"""Daily retention for task_run rows. FC-3i.
- 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
Success is high-volume, not interesting after a day.
- 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
(7 days default). Failures are operationally interesting longer.
- 'running' rows: NEVER deleted by this task. The recovery sweep
(recover_stalled_task_runs) is the mechanism that flips them to
terminal state; prune doesn't touch in-flight state.
- 'retry' rows: treated as failures (>7d).
Returns dict of how many rows were deleted in each bucket.
"""
SessionLocal = _sync_session_factory()
now = datetime.now(UTC)
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
with SessionLocal() as session:
ok_deleted = session.execute(
delete(TaskRun)
.where(TaskRun.status == "ok")
.where(TaskRun.finished_at < ok_cutoff)
).rowcount or 0
fail_deleted = session.execute(
delete(TaskRun)
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
.where(TaskRun.finished_at < fail_cutoff)
).rowcount or 0
session.commit()
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
@celery.task(name="backend.app.tasks.maintenance.backfill_phash") @celery.task(name="backend.app.tasks.maintenance.backfill_phash")
def backfill_phash() -> int: def backfill_phash() -> int:
"""Recompute phash for stored images that have none (imported before """Recompute phash for stored images that have none (imported before
+26 -14
View File
@@ -2,9 +2,6 @@
<Teleport to="body"> <Teleport to="body">
<div <div
class="fc-viewer" role="dialog" aria-modal="true" class="fc-viewer" role="dialog" aria-modal="true"
@keydown.esc="$emit('close')"
@keydown.left="onArrowLeft"
@keydown.right="onArrowRight"
tabindex="-1" ref="rootEl" tabindex="-1" ref="rootEl"
> >
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close"> <button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
@@ -68,7 +65,7 @@ import VideoCanvas from './VideoCanvas.vue'
import TagPanel from './TagPanel.vue' import TagPanel from './TagPanel.vue'
import ProvenancePanel from './ProvenancePanel.vue' import ProvenancePanel from './ProvenancePanel.vue'
defineEmits(['close']) const emit = defineEmits(['close'])
const modal = useModalStore() const modal = useModalStore()
const rootEl = ref(null) const rootEl = ref(null)
@@ -86,14 +83,39 @@ const integrityBadge = computed(() => {
}) })
let prevBodyOverflow = null let prevBodyOverflow = null
// Document-level keyboard handler. Bound on document (not on the modal
// root) because <video> elements capture focus when the user interacts
// with their controls — when video has focus, Escape and arrow keys
// don't bubble reliably to ancestors. Listening on document side-steps
// that. Filtered via isTextEntry so tag/comment inputs still get their
// own keystrokes.
function onKeyDown(ev) {
if (ev.key === 'Escape') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
emit('close')
} else if (ev.key === 'ArrowLeft') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goPrev()
} else if (ev.key === 'ArrowRight') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goNext()
}
}
onMounted(async () => { onMounted(async () => {
prevBodyOverflow = document.body.style.overflow prevBodyOverflow = document.body.style.overflow
document.body.style.overflow = 'hidden' document.body.style.overflow = 'hidden'
document.addEventListener('keydown', onKeyDown, true)
await nextFrame() await nextFrame()
rootEl.value?.focus() rootEl.value?.focus()
}) })
onUnmounted(() => { onUnmounted(() => {
document.body.style.overflow = prevBodyOverflow document.body.style.overflow = prevBodyOverflow
document.removeEventListener('keydown', onKeyDown, true)
}) })
watch(() => modal.currentImageId, async () => { watch(() => modal.currentImageId, async () => {
@@ -105,16 +127,6 @@ function nextFrame() {
return new Promise(resolve => requestAnimationFrame(resolve)) return new Promise(resolve => requestAnimationFrame(resolve))
} }
function onArrowLeft(ev) {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goPrev()
}
function onArrowRight(ev) {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goNext()
}
function isTextEntry(el) { function isTextEntry(el) {
if (!el) return false if (!el) return false
const tag = el.tagName const tag = el.tagName
@@ -43,6 +43,21 @@
class="fc-prov__desc" v-html="e.post.description_html" class="fc-prov__desc" v-html="e.post.description_html"
/> />
</article> </article>
<!-- Fallback: image has artist_id set (folder-derived from
filesystem import or set by tag_apply) but no ImageProvenance
row exists. Show a minimal "by <artist>" line so the modal
isn't blank for IR-imported images that aren't in any post. -->
<article
v-if="showArtistFallback"
class="fc-prov__card"
>
<div class="fc-prov__meta">
<RouterLink :to="`/artist/${fallbackArtist.slug}`">
by {{ fallbackArtist.name }}
</RouterLink>
</div>
</article>
</template> </template>
<div v-if="attachments.length" class="fc-prov__attach"> <div v-if="attachments.length" class="fc-prov__attach">
@@ -79,13 +94,27 @@ const state = computed(() =>
modal.currentImageId == null ? null : prov.imageProv(modal.currentImageId) modal.currentImageId == null ? null : prov.imageProv(modal.currentImageId)
) )
// Show the panel while loading or on error, or when there is >=1 entry. const fallbackArtist = computed(() => modal.current?.artist || null)
// Hide entirely when the image has zero provenance.
const showArtistFallback = computed(() => {
const st = state.value
// Only show the fallback when provenance is genuinely empty (no
// entries, not loading, no error) AND the image has a direct
// artist_id we can surface.
if (!st || st.loading || st.error) return false
if (Array.isArray(st.entries) && st.entries.length > 0) return false
return fallbackArtist.value != null
})
// Show the panel while loading or on error, when there is >=1 entry,
// or when the artist fallback applies (image_record.artist_id set but
// no ImageProvenance row, e.g. filesystem-imported IR images).
const show = computed(() => { const show = computed(() => {
const st = state.value const st = state.value
if (!st) return false if (!st) return false
if (st.loading || st.error) return true if (st.loading || st.error) return true
return Array.isArray(st.entries) && st.entries.length > 0 if (Array.isArray(st.entries) && st.entries.length > 0) return true
return showArtistFallback.value
}) })
const attachments = computed(() => state.value?.attachments || []) const attachments = computed(() => state.value?.attachments || [])
@@ -0,0 +1,109 @@
<template>
<v-table density="compact" class="fc-queues-table">
<thead>
<tr>
<th>Queue</th>
<th class="text-right">Depth</th>
<th class="text-right">Workers</th>
<th v-if="!compact" class="text-right">Active</th>
<th class="text-right">Last min: ok / err</th>
</tr>
</thead>
<tbody>
<tr v-for="q in QUEUE_NAMES" :key="q">
<td>{{ q }}</td>
<td class="text-right fc-tabular" :class="depthClass(q)">
{{ formatDepth(q) }}
</td>
<td class="text-right fc-tabular">{{ workerCount(q) }}</td>
<td v-if="!compact" class="text-right fc-tabular">{{ activeCount(q) }}</td>
<td class="text-right fc-tabular">
<span class="fc-ok">{{ recentOk(q) }}</span>
<span class="fc-sep">/</span>
<span :class="recentErr(q) > 0 ? 'fc-err' : 'fc-muted'">
{{ recentErr(q) }}<span v-if="recentErr(q) > 0"> </span>
</span>
</td>
</tr>
</tbody>
</v-table>
</template>
<script setup>
import { computed } from 'vue'
const props = defineProps({
queues: { type: Object, default: null }, // store.queues
workers: { type: Object, default: null }, // store.workers
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
compact: { type: Boolean, default: false },
})
const QUEUE_NAMES = [
'default', 'import', 'thumbnail', 'ml',
'download', 'scan', 'maintenance',
]
function formatDepth(name) {
const d = props.queues?.queues?.[name]
if (d == null) return '—'
return d.toLocaleString()
}
function depthClass(name) {
const d = props.queues?.queues?.[name]
const workers = workerCount(name)
// Queue has depth but no workers → warning.
if (d != null && d > 0 && workers === 0) return 'fc-warn'
return ''
}
function workerCount(name) {
if (!props.workers?.workers) return 0
let count = 0
for (const info of Object.values(props.workers.workers)) {
if (info.queues?.includes(name)) count++
}
return count
}
function activeCount(name) {
if (!props.workers?.workers) return 0
// Active count is per-worker, not per-queue. Approximate: sum of
// active_count across workers that subscribe to this queue.
let count = 0
for (const info of Object.values(props.workers.workers)) {
if (info.queues?.includes(name)) count += info.active_count || 0
}
return count
}
const recentByQueue = computed(() => {
const out = {}
for (const r of props.recentMinute) {
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
if (r.status === 'ok') out[r.queue].ok++
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
}
return out
})
function recentOk(name) { return recentByQueue.value[name]?.ok || 0 }
function recentErr(name) { return recentByQueue.value[name]?.err || 0 }
</script>
<style scoped>
.fc-queues-table { background: transparent; }
.fc-tabular {
font-variant-numeric: tabular-nums;
font-feature-settings: 'tnum';
}
.fc-ok { color: rgb(var(--v-theme-success, 76 175 80)); }
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
.fc-warn { color: rgb(var(--v-theme-warning, 255 179 0)); font-weight: 500; }
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
.fc-sep {
margin: 0 4px;
color: rgb(var(--v-theme-on-surface-variant));
}
</style>
@@ -0,0 +1,64 @@
<template>
<v-card class="fc-activity-summary">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-pulse" size="small" />
<span>System activity (last min)</span>
<v-spacer />
<v-btn
variant="text" size="small" rounded="pill"
@click="$emit('open-activity')"
>
View activity
<v-icon end size="small">mdi-arrow-right</v-icon>
</v-btn>
</v-card-title>
<v-card-text>
<v-alert
v-if="store.lastError"
type="warning" variant="tonal" density="compact" class="mb-3"
>
{{ store.lastError }}
</v-alert>
<QueuesTable
:queues="store.queues"
:workers="store.workers"
:recent-minute="store.recentMinute"
compact
/>
</v-card-text>
</v-card>
</template>
<script setup>
import { onMounted, onUnmounted } from 'vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
import QueuesTable from './QueuesTable.vue'
defineEmits(['open-activity'])
const store = useSystemActivityStore()
let pollId = null
function pollOnce() {
if (document.hidden) return
store.loadQueues()
store.loadWorkers()
store.loadRecentMinute()
}
onMounted(() => {
pollOnce()
pollId = setInterval(pollOnce, 5000)
})
onUnmounted(() => {
if (pollId) { clearInterval(pollId); pollId = null }
})
</script>
<style scoped>
.fc-activity-summary { border-radius: 8px; }
</style>
@@ -0,0 +1,280 @@
<template>
<div class="fc-activity">
<!-- Queues + workers pane -->
<v-card class="mb-4">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-pulse" size="small" />
<span>Queues + workers</span>
<v-spacer />
<span class="text-caption fc-muted">
updated {{ formatRelative(store.queues?.fetched_at) }}
</span>
</v-card-title>
<v-card-text>
<QueuesTable
:queues="store.queues"
:workers="store.workers"
:recent-minute="store.recentMinute"
/>
</v-card-text>
</v-card>
<!-- Recent failures pane -->
<v-card class="mb-4">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-alert-circle-outline" size="small" />
<span>Recent failures (last 24h)</span>
<v-spacer />
<span class="text-caption fc-muted">
{{ failureCount }} failures
</span>
</v-card-title>
<v-card-text>
<div v-if="errorTypes.length" class="mb-3 fc-pills">
<v-chip
v-for="t in errorTypes" :key="t.name"
:color="t.name === filterErrorType ? 'accent' : undefined"
size="small" variant="tonal" closable
@click="toggleErrorTypeFilter(t.name)"
@click:close="toggleErrorTypeFilter(t.name)"
>
{{ t.name }} × {{ t.count }}
</v-chip>
</div>
<v-table density="compact">
<thead>
<tr>
<th>Time</th>
<th>Queue</th>
<th>Task</th>
<th>Target</th>
<th>Error</th>
</tr>
</thead>
<tbody>
<tr v-for="r in filteredFailures" :key="r.id">
<td class="fc-tabular" :title="r.finished_at">
{{ formatRelative(r.finished_at) }}
</td>
<td>{{ r.queue }}</td>
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
<td class="fc-err" :title="r.error_message">
{{ r.error_type }}
</td>
</tr>
<tr v-if="!filteredFailures.length">
<td colspan="5" class="text-center fc-muted py-4">
No failures in the last 24h.
</td>
</tr>
</tbody>
</v-table>
</v-card-text>
</v-card>
<!-- All activity pane -->
<v-card>
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-format-list-bulleted" size="small" />
<span>All recent activity</span>
<v-spacer />
<v-select
v-model="filterQueue"
:items="queueOptions" density="compact" hide-details
style="max-width: 180px;"
@update:model-value="onFilterChange"
/>
<v-select
v-model="filterStatus"
:items="statusOptions" density="compact" hide-details
style="max-width: 180px;"
@update:model-value="onFilterChange"
/>
<v-btn
variant="text" size="small" rounded="pill"
@click="onRefresh"
>
<v-icon start size="small">mdi-refresh</v-icon>
Refresh
</v-btn>
</v-card-title>
<v-card-text>
<v-table density="compact">
<thead>
<tr>
<th></th>
<th>Time</th>
<th>Queue</th>
<th>Task</th>
<th>Target</th>
<th class="text-right">Duration</th>
<th>Status</th>
</tr>
</thead>
<tbody>
<tr v-for="r in store.runs" :key="r.id">
<td><v-icon size="small" :color="statusColor(r.status)">{{ statusIcon(r.status) }}</v-icon></td>
<td class="fc-tabular" :title="r.started_at">{{ formatRelative(r.started_at) }}</td>
<td>{{ r.queue }}</td>
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
<td class="text-right fc-tabular">{{ formatDuration(r.duration_ms) }}</td>
<td>{{ r.status }}</td>
</tr>
<tr v-if="!store.runs.length && !store.loading.runs">
<td colspan="7" class="text-center fc-muted py-4">
No activity yet.
</td>
</tr>
</tbody>
</v-table>
<div v-if="store.runsHasMore" class="d-flex justify-center py-3">
<v-btn
variant="text" size="small"
:loading="store.loading.runs"
@click="onLoadMore"
>Load more</v-btn>
</div>
</v-card-text>
</v-card>
</div>
</template>
<script setup>
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
import QueuesTable from './QueuesTable.vue'
const store = useSystemActivityStore()
const filterQueue = ref(null)
const filterStatus = ref(null)
const filterErrorType = ref(null)
const queueOptions = [
{ title: 'All queues', value: null },
{ title: 'import', value: 'import' },
{ title: 'thumbnail', value: 'thumbnail' },
{ title: 'ml', value: 'ml' },
{ title: 'download', value: 'download' },
{ title: 'scan', value: 'scan' },
{ title: 'maintenance', value: 'maintenance' },
{ title: 'default', value: 'default' },
]
const statusOptions = [
{ title: 'All statuses', value: null },
{ title: 'Running', value: 'running' },
{ title: 'OK', value: 'ok' },
{ title: 'Error', value: 'error' },
{ title: 'Timeout', value: 'timeout' },
{ title: 'Retry', value: 'retry' },
]
let queuesPollId = null
let failuresPollId = null
function pollQueues() {
if (document.hidden) return
store.loadQueues()
store.loadWorkers()
store.loadRecentMinute()
}
function pollFailures() {
if (document.hidden) return
store.loadFailures()
}
onMounted(() => {
pollQueues()
pollFailures()
store.setFilter({ queue: null, status: null })
store.loadRuns({ reset: true })
queuesPollId = setInterval(pollQueues, 3000)
failuresPollId = setInterval(pollFailures, 15000)
})
onUnmounted(() => {
if (queuesPollId) clearInterval(queuesPollId)
if (failuresPollId) clearInterval(failuresPollId)
})
const errorTypes = computed(() => {
const counts = store.failures?.count_by_type || {}
return Object.entries(counts).map(([name, count]) => ({ name, count }))
})
const failureCount = computed(() => (store.failures?.recent || []).length)
const filteredFailures = computed(() => {
const all = store.failures?.recent || []
if (!filterErrorType.value) return all
return all.filter(r => r.error_type === filterErrorType.value)
})
function toggleErrorTypeFilter(name) {
filterErrorType.value = filterErrorType.value === name ? null : name
if (filterErrorType.value) {
// Mirror into the All-activity table too.
filterStatus.value = 'error'
onFilterChange()
}
}
function onFilterChange() {
store.setFilter({ queue: filterQueue.value, status: filterStatus.value })
store.loadRuns({ reset: true })
}
function onLoadMore() { store.loadRuns({ reset: false }) }
function onRefresh() { store.loadRuns({ reset: true }) }
function statusIcon(status) {
return {
ok: 'mdi-check-circle',
error: 'mdi-close-circle',
timeout: 'mdi-clock-alert',
retry: 'mdi-refresh',
running: 'mdi-timer-sand',
}[status] || 'mdi-help-circle'
}
function statusColor(status) {
return {
ok: 'success', error: 'error', timeout: 'warning',
retry: 'info', running: 'accent',
}[status] || 'on-surface-variant'
}
function shortTaskName(name) {
if (!name) return ''
const parts = name.split('.')
return parts.slice(-1)[0]
}
function formatDuration(ms) {
if (ms == null) return '—'
if (ms < 1000) return `${ms} ms`
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
return `${(ms / 60_000).toFixed(1)} min`
}
function formatRelative(iso) {
if (!iso) return '—'
const then = new Date(iso).getTime()
const now = Date.now()
const diff = Math.max(0, (now - then) / 1000)
if (diff < 60) return `${Math.floor(diff)}s ago`
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
return `${Math.floor(diff / 86400)}d ago`
}
</script>
<style scoped>
.fc-activity { padding: 0; }
.fc-pills {
display: flex; gap: 6px; flex-wrap: wrap;
}
.fc-tabular {
font-variant-numeric: tabular-nums;
font-feature-settings: 'tnum';
}
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
</style>
+103
View File
@@ -0,0 +1,103 @@
import { defineStore } from 'pinia'
import { ref } from 'vue'
import { useApi } from '../composables/useApi.js'
export const useSystemActivityStore = defineStore('systemActivity', () => {
const api = useApi()
// Live polled state.
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
const recentMinute = ref([]) // last-60s rows (for Overview summary)
const failures = ref(null) // { recent, count_by_type, since }
// Paginated runs (Activity tab "All recent activity" pane).
const runs = ref([])
const runsCursor = ref(null)
const runsHasMore = ref(false)
const runsFilter = ref({ queue: null, status: null, limit: 50 })
const loading = ref({ queues: false, workers: false, runs: false, failures: false })
const lastError = ref(null)
async function loadQueues() {
loading.value.queues = true
lastError.value = null
try {
queues.value = await api.get('/api/system/activity/queues')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.queues = false
}
}
async function loadWorkers() {
loading.value.workers = true
lastError.value = null
try {
workers.value = await api.get('/api/system/activity/workers')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.workers = false
}
}
async function loadRecentMinute() {
// Used by the Overview summary card: pull last 60s of runs to compute
// per-queue ok/err counts. One call covers all queues; UI groups.
try {
const body = await api.get('/api/system/activity/runs', {
params: { limit: 200 },
})
recentMinute.value = body.runs || []
} catch (e) {
lastError.value = e.message
}
}
async function loadRuns({ reset = false } = {}) {
loading.value.runs = true
lastError.value = null
try {
const params = { limit: runsFilter.value.limit }
if (runsFilter.value.queue) params.queue = runsFilter.value.queue
if (runsFilter.value.status) params.status = runsFilter.value.status
if (!reset && runsCursor.value) params.before_id = runsCursor.value
const body = await api.get('/api/system/activity/runs', { params })
runs.value = reset ? body.runs : [...runs.value, ...body.runs]
runsCursor.value = body.next_cursor
runsHasMore.value = body.next_cursor !== null
} catch (e) {
lastError.value = e.message
} finally {
loading.value.runs = false
}
}
async function loadFailures() {
loading.value.failures = true
lastError.value = null
try {
failures.value = await api.get('/api/system/activity/failures')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.failures = false
}
}
function setFilter(filter) {
runsFilter.value = { ...runsFilter.value, ...filter }
}
return {
queues, workers, recentMinute, failures,
runs, runsCursor, runsHasMore, runsFilter,
loading, lastError,
loadQueues, loadWorkers, loadRecentMinute,
loadRuns, loadFailures, setFilter,
}
})
+11
View File
@@ -2,6 +2,7 @@
<v-container fluid class="py-6"> <v-container fluid class="py-6">
<v-tabs v-model="tab" color="accent" class="mb-4"> <v-tabs v-model="tab" color="accent" class="mb-4">
<v-tab value="overview">Overview</v-tab> <v-tab value="overview">Overview</v-tab>
<v-tab value="activity">Activity</v-tab>
<v-tab value="import">Import</v-tab> <v-tab value="import">Import</v-tab>
<v-tab value="maintenance">Maintenance</v-tab> <v-tab value="maintenance">Maintenance</v-tab>
</v-tabs> </v-tabs>
@@ -9,6 +10,10 @@
<v-window v-model="tab"> <v-window v-model="tab">
<v-window-item value="overview"> <v-window-item value="overview">
<SystemStatsCards :stats="system.stats" /> <SystemStatsCards :stats="system.stats" />
<SystemActivitySummary
class="mt-4"
@open-activity="tab = 'activity'"
/>
<v-alert <v-alert
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0" v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
type="info" variant="tonal" class="mt-4" closable type="info" variant="tonal" class="mt-4" closable
@@ -18,6 +23,10 @@
</v-alert> </v-alert>
</v-window-item> </v-window-item>
<v-window-item value="activity">
<SystemActivityTab />
</v-window-item>
<v-window-item value="import"> <v-window-item value="import">
<ImportTriggerPanel /> <ImportTriggerPanel />
<v-divider class="my-6" /> <v-divider class="my-6" />
@@ -38,6 +47,8 @@ import { onMounted, onUnmounted, ref, watch } from 'vue'
import { useSystemStore } from '../stores/system.js' import { useSystemStore } from '../stores/system.js'
import { useImportStore } from '../stores/import.js' import { useImportStore } from '../stores/import.js'
import SystemStatsCards from '../components/settings/SystemStatsCards.vue' import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue' import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue' import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
import ImportTaskList from '../components/settings/ImportTaskList.vue' import ImportTaskList from '../components/settings/ImportTaskList.vue'
+5 -18
View File
@@ -83,24 +83,11 @@ async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
assert resp.status_code == 202 assert resp.status_code == 202
@pytest.mark.asyncio # Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
async def test_post_apply_without_backup_rejected(client, monkeypatch): # the migration backup-gate rejected applies without a recent backup.
monkeypatch.setattr( # That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
"backend.app.api.migrate._has_recent_pre_migration_backup", # FC-3h will own backup as a first-class feature) so the test was
lambda: False, # testing dead behavior. Removed rather than rewritten.
)
export = {
"source_app": "gallerysubscriber", "schema_version": 1,
"subscriptions": [], "credentials": [],
}
resp = await client.post(
"/api/migrate/gs_ingest",
form={"dry_run": "false"},
files={"export_file": _file_storage(export, "gs-export.json")},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "no_backup"
@pytest.mark.asyncio @pytest.mark.asyncio
+267
View File
@@ -0,0 +1,267 @@
"""FC-3i: /api/system/activity/* endpoint tests.
Mocks Redis LLEN + celery inspect via monkeypatch on the module-level
_read_queues_sync / _read_workers_sync helpers, so tests don't need
a live broker or running workers.
"""
from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from backend.app import create_app
from backend.app.api import system_activity as activity_module
from backend.app.models import TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture
async def app():
return create_app()
@pytest.fixture
async def client(app):
async with app.test_client() as c:
yield c
@pytest.fixture(autouse=True)
def _reset_caches(monkeypatch):
"""Clear the module-level caches between tests so cache state from
one test doesn't leak into the next."""
monkeypatch.setitem(activity_module._QUEUE_CACHE, "data", None)
monkeypatch.setitem(activity_module._QUEUE_CACHE, "ts", 0.0)
monkeypatch.setitem(activity_module._WORKER_CACHE, "data", None)
monkeypatch.setitem(activity_module._WORKER_CACHE, "ts", 0.0)
# --- /queues -------------------------------------------------------
@pytest.mark.asyncio
async def test_queues_returns_all_known_queues(client, monkeypatch):
def _fake():
return {
"queues": dict.fromkeys(activity_module._QUEUE_NAMES, 0),
"fetched_at": datetime.now(UTC).isoformat(),
}
monkeypatch.setattr(activity_module, "_read_queues_sync", _fake)
resp = await client.get("/api/system/activity/queues")
assert resp.status_code == 200
body = await resp.get_json()
assert set(body["queues"].keys()) == set(activity_module._QUEUE_NAMES)
@pytest.mark.asyncio
async def test_queues_cached_within_ttl(client, monkeypatch):
call_count = {"n": 0}
def _counting():
call_count["n"] += 1
return {"queues": {}, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_queues_sync", _counting)
await client.get("/api/system/activity/queues")
await client.get("/api/system/activity/queues")
await client.get("/api/system/activity/queues")
assert call_count["n"] == 1 # cached after the first call
@pytest.mark.asyncio
async def test_queues_redis_failure_returns_null_per_failing_queue(client, monkeypatch):
"""_read_queues_sync's per-queue try/except returns None on failure."""
def _partial():
out = dict.fromkeys(activity_module._QUEUE_NAMES, 0)
out["ml"] = None # simulate broker hiccup on the ml queue
return {"queues": out, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_queues_sync", _partial)
resp = await client.get("/api/system/activity/queues")
body = await resp.get_json()
assert body["queues"]["ml"] is None
assert body["queues"]["import"] == 0
# --- /workers ------------------------------------------------------
@pytest.mark.asyncio
async def test_workers_returns_per_worker_queue_membership(client, monkeypatch):
def _fake():
return {
"workers": {
"worker@host1": {"queues": ["import", "thumbnail"], "active_count": 2},
"worker@host2": {"queues": ["ml"], "active_count": 0},
},
"fetched_at": "x",
}
monkeypatch.setattr(activity_module, "_read_workers_sync", _fake)
resp = await client.get("/api/system/activity/workers")
assert resp.status_code == 200
body = await resp.get_json()
assert "worker@host1" in body["workers"]
assert "import" in body["workers"]["worker@host1"]["queues"]
@pytest.mark.asyncio
async def test_workers_cached_within_ttl(client, monkeypatch):
call_count = {"n": 0}
def _counting():
call_count["n"] += 1
return {"workers": {}, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_workers_sync", _counting)
await client.get("/api/system/activity/workers")
await client.get("/api/system/activity/workers")
assert call_count["n"] == 1
# --- /runs ---------------------------------------------------------
@pytest_asyncio.fixture
async def _seed_runs(db):
"""Insert 5 task_run rows for paging tests."""
now = datetime.now(UTC)
for i in range(5):
db.add(TaskRun(
celery_task_id=f"tid-{i}",
queue="import" if i % 2 == 0 else "ml",
task_name=f"backend.app.tasks.fake.task_{i}",
target_id=i,
started_at=now - timedelta(seconds=10 - i),
finished_at=now - timedelta(seconds=9 - i),
duration_ms=1000,
status="ok" if i < 3 else "error",
error_type="ValueError" if i >= 3 else None,
error_message="boom" if i >= 3 else None,
))
await db.commit()
@pytest.mark.asyncio
async def test_runs_paginated_descending_by_id(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?limit=3")
body = await resp.get_json()
assert len(body["runs"]) == 3
# Descending: latest id first.
ids = [r["id"] for r in body["runs"]]
assert ids == sorted(ids, reverse=True)
assert body["next_cursor"] is not None # 5 total, asked for 3 → more
@pytest.mark.asyncio
async def test_runs_filter_by_queue(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?queue=ml")
body = await resp.get_json()
assert all(r["queue"] == "ml" for r in body["runs"])
assert len(body["runs"]) == 2 # i=1, i=3
@pytest.mark.asyncio
async def test_runs_filter_by_status(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?status=error")
body = await resp.get_json()
assert all(r["status"] == "error" for r in body["runs"])
assert len(body["runs"]) == 2 # i=3, i=4
@pytest.mark.asyncio
async def test_runs_keyset_cursor(client, _seed_runs):
page1 = await (await client.get("/api/system/activity/runs?limit=2")).get_json()
assert len(page1["runs"]) == 2
assert page1["next_cursor"] is not None
page2 = await (await client.get(
f"/api/system/activity/runs?limit=2&before_id={page1['next_cursor']}"
)).get_json()
assert len(page2["runs"]) == 2
page1_ids = {r["id"] for r in page1["runs"]}
page2_ids = {r["id"] for r in page2["runs"]}
assert page1_ids.isdisjoint(page2_ids)
@pytest.mark.asyncio
async def test_runs_invalid_limit_400(client):
resp = await client.get("/api/system/activity/runs?limit=not-a-number")
assert resp.status_code == 400
# --- /failures -----------------------------------------------------
@pytest_asyncio.fixture
async def _seed_failures(db):
"""Mix of ok / error / timeout rows; some old, some recent."""
now = datetime.now(UTC)
recent = [
("error", "OperationalError"),
("error", "OperationalError"),
("error", "OperationalError"),
("timeout", "TimeoutError"),
("timeout", "TimeoutError"),
("error", "OSError"),
("ok", None), # success — should NOT appear in failures
]
for i, (status, etype) in enumerate(recent):
db.add(TaskRun(
celery_task_id=f"f-{i}",
queue="ml",
task_name="backend.app.tasks.fake.x",
target_id=i,
started_at=now - timedelta(seconds=120 - i),
finished_at=now - timedelta(seconds=60 - i),
duration_ms=1000,
status=status,
error_type=etype,
error_message="boom" if status != "ok" else None,
))
# One ancient failure (>24h) that should NOT appear.
db.add(TaskRun(
celery_task_id="f-old",
queue="ml",
task_name="backend.app.tasks.fake.x",
target_id=999,
started_at=now - timedelta(hours=30),
finished_at=now - timedelta(hours=29),
duration_ms=1000,
status="error",
error_type="OldError",
error_message="ancient",
))
await db.commit()
@pytest.mark.asyncio
async def test_failures_returns_recent_errors_and_timeouts(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
statuses = {r["status"] for r in body["recent"]}
assert statuses == {"error", "timeout"}
# No ok rows.
assert "ok" not in statuses
@pytest.mark.asyncio
async def test_failures_count_by_type_groups_correctly(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
counts = body["count_by_type"]
assert counts.get("OperationalError") == 3
assert counts.get("TimeoutError") == 2
assert counts.get("OSError") == 1
assert "OldError" not in counts # outside 24h window
@pytest.mark.asyncio
async def test_failures_only_within_24h_window(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
ids = {r["error_type"] for r in body["recent"]}
assert "OldError" not in ids
+171
View File
@@ -93,3 +93,174 @@ def test_cleanup_old_deletes_finished_old(db_sync):
remaining = {t.source_path for t in db_sync.query(ImportTask).all()} remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
assert remaining == {"/import/b.jpg", "/import/c.jpg"} assert remaining == {"/import/b.jpg", "/import/c.jpg"}
# --- FC-3i: task_run sweep + retention -----------------------------
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
error_type=None):
from backend.app.models import TaskRun
row = TaskRun(
celery_task_id="x",
queue="ml",
task_name="backend.app.tasks.fake.t",
target_id=1,
started_at=started_at,
finished_at=finished_at,
duration_ms=1000 if finished_at else None,
status=status,
error_type=error_type,
error_message="x" if status in ("error", "timeout") else None,
)
db_sync.add(row)
db_sync.flush()
return row.id
def test_recover_stalled_task_runs_flips_old_running_to_error(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import recover_stalled_task_runs
now = datetime.now(UTC)
stale_id = _make_task_run(
db_sync, status="running", started_at=now - timedelta(minutes=10),
)
db_sync.commit()
recovered = recover_stalled_task_runs.apply().get()
assert recovered == 1
db_sync.expire_all()
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.id == stale_id)
).scalar_one()
error_type = db_sync.execute(
select(TaskRun.error_type).where(TaskRun.id == stale_id)
).scalar_one()
assert status == "error"
assert error_type == "RecoverySweep"
def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import recover_stalled_task_runs
now = datetime.now(UTC)
fresh_id = _make_task_run(
db_sync, status="running", started_at=now - timedelta(seconds=30),
)
db_sync.commit()
recovered = recover_stalled_task_runs.apply().get()
assert recovered == 0
db_sync.expire_all()
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.id == fresh_id)
).scalar_one()
assert status == "running"
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
old_id = _make_task_run(
db_sync, status="ok",
started_at=now - timedelta(hours=30),
finished_at=now - timedelta(hours=29),
)
recent_id = _make_task_run(
db_sync, status="ok",
started_at=now - timedelta(hours=2),
finished_at=now - timedelta(hours=1),
)
db_sync.commit()
result = prune_task_runs.apply().get()
assert result["ok_deleted"] == 1
db_sync.expire_all()
surviving_ids = set(db_sync.execute(
select(TaskRun.id).where(TaskRun.id.in_([old_id, recent_id]))
).scalars().all())
assert surviving_ids == {recent_id}
def test_prune_task_runs_deletes_failures_older_than_7d(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
old_id = _make_task_run(
db_sync, status="error", error_type="OldError",
started_at=now - timedelta(days=10),
finished_at=now - timedelta(days=9),
)
db_sync.commit()
result = prune_task_runs.apply().get()
assert result["failures_deleted"] >= 1
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == old_id)
).scalar_one_or_none()
assert surviving is None
def test_prune_task_runs_keeps_recent_failures(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
recent_id = _make_task_run(
db_sync, status="error", error_type="RecentError",
started_at=now - timedelta(days=6),
finished_at=now - timedelta(days=5),
)
db_sync.commit()
prune_task_runs.apply().get()
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == recent_id)
).scalar_one_or_none()
assert surviving == recent_id
def test_prune_task_runs_never_deletes_running(db_sync):
"""Even a 30-day-old running row stays — recovery sweep is the
mechanism that flips them; prune doesn't touch in-flight."""
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
ancient_id = _make_task_run(
db_sync, status="running",
started_at=now - timedelta(days=30),
)
db_sync.commit()
prune_task_runs.apply().get()
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == ancient_id)
).scalar_one_or_none()
assert surviving == ancient_id
+146 -4
View File
@@ -2,9 +2,19 @@
import json import json
import pytest import pytest
from sqlalchemy import select from sqlalchemy import func, select
from backend.app.models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag from backend.app.models import (
Artist,
ImageProvenance,
ImageRecord,
Post,
SeriesPage,
Source,
Tag,
TagKind,
image_tag,
)
from backend.app.services.migrators import tag_apply from backend.app.services.migrators import tag_apply
pytestmark = pytest.mark.integration pytestmark = pytest.mark.integration
@@ -21,14 +31,19 @@ async def _seed_image(db, sha: str, *, suffix="x"):
return img return img
def _write_manifest(tmp_path, *, artist_assignments=None, tag_associations=None, series_pages=None): def _write_manifest(
tmp_path, *,
artist_assignments=None, tag_associations=None,
series_pages=None, image_posts=None, schema_version=2,
):
d = tmp_path / "_migration_state" d = tmp_path / "_migration_state"
d.mkdir(parents=True, exist_ok=True) d.mkdir(parents=True, exist_ok=True)
(d / "ir_tag_manifest.json").write_text(json.dumps({ (d / "ir_tag_manifest.json").write_text(json.dumps({
"schema_version": 1, "schema_version": schema_version,
"image_artist_assignments": artist_assignments or [], "image_artist_assignments": artist_assignments or [],
"image_tag_associations": tag_associations or [], "image_tag_associations": tag_associations or [],
"series_pages": series_pages or [], "series_pages": series_pages or [],
"image_posts": image_posts or [],
})) }))
@@ -154,3 +169,130 @@ async def test_dry_run_makes_no_changes(db, tmp_path):
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
)).all() )).all()
assert len(rows) == 0 assert len(rows) == 0
# --- Phase 4: image_posts (schema v2) -----------------------------------
_POST_ENTRY = {
"platform": "patreon",
"post_id": "10001",
"artist": "Maewix",
"title": "Test Post Title",
"description": "<p>HTML description</p>",
"source_url": "https://www.patreon.com/posts/test-10001",
"attachment_count": 1,
"published_at": "2026-01-15T12:00:00+00:00",
}
@pytest.mark.asyncio
async def test_image_posts_creates_source_post_provenance(db, tmp_path):
sha = "d" * 64
await _seed_image(db, sha, suffix="d")
await db.commit()
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "image_sha256s": [sha]},
])
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
assert result["counts"]["rows_inserted"] >= 1
# Async Core-DML assertions — column selects, not ORM attribute access.
artist_count = (await db.execute(
select(func.count(Artist.id)).where(Artist.slug == "maewix")
)).scalar_one()
assert artist_count == 1
source_count = (await db.execute(
select(func.count(Source.id)).where(
Source.platform == "patreon",
Source.url == "https://www.patreon.com/maewix",
)
)).scalar_one()
assert source_count == 1
post_count = (await db.execute(
select(func.count(Post.id)).where(Post.external_post_id == "10001")
)).scalar_one()
assert post_count == 1
img_id = (await db.execute(
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
)).scalar_one()
prov_count = (await db.execute(
select(func.count(ImageProvenance.id))
.where(ImageProvenance.image_record_id == img_id)
)).scalar_one()
assert prov_count == 1
@pytest.mark.asyncio
async def test_image_posts_idempotent_on_rerun(db, tmp_path):
sha = "e" * 64
await _seed_image(db, sha, suffix="e")
await db.commit()
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "image_sha256s": [sha]},
])
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
# Re-run: nothing should be newly inserted; all skipped.
assert result2["counts"]["rows_inserted"] == 0
assert result2["counts"]["rows_skipped"] >= 1
prov_count = (await db.execute(
select(func.count(ImageProvenance.id))
)).scalar_one()
assert prov_count == 1
@pytest.mark.asyncio
async def test_image_posts_unknown_sha_unmatched(db, tmp_path):
# No image seeded; sha doesn't exist.
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "image_sha256s": ["f" * 64]},
])
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
unmatched_kinds = [u["kind"] for u in result["unmatched"]]
assert "post" in unmatched_kinds
@pytest.mark.asyncio
async def test_image_posts_unknown_platform_skipped(db, tmp_path):
sha = "1" * 64
await _seed_image(db, sha, suffix="1")
await db.commit()
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "platform": "tumblr", "image_sha256s": [sha]},
])
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
assert result["counts"]["rows_skipped"] >= 1
# No Post / Source / ImageProvenance should have been created.
src_count = (await db.execute(
select(func.count(Source.id)).where(Source.platform == "tumblr")
)).scalar_one()
assert src_count == 0
@pytest.mark.asyncio
async def test_image_posts_dry_run_makes_no_writes(db, tmp_path):
sha = "2" * 64
await _seed_image(db, sha, suffix="2")
await db.commit()
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "image_sha256s": [sha]},
])
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
prov_count = (await db.execute(
select(func.count(ImageProvenance.id))
)).scalar_one()
assert prov_count == 0
+258
View File
@@ -0,0 +1,258 @@
"""FC-3i: Celery signal → task_run row tests.
Uses task_always_eager so signals fire synchronously in-process during
the test. Scoped autouse fixture keeps the eager mode confined to this
file (other tests may break with eager mode enabled globally).
Test assertions on signal-mutated rows use COLUMN SELECTS / db_sync.
scalar (per reference_async_coredml_test_assertions). ORM attribute
access on a row a signal handler just mutated risks MissingGreenlet
across async/sync boundaries.
"""
import pytest
from sqlalchemy import func, select
from backend.app.celery_app import celery
from backend.app.models import TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture(autouse=True)
def _eager_celery(monkeypatch):
"""task_always_eager so signals fire in-process during the test."""
monkeypatch.setattr(celery.conf, "task_always_eager", True)
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
@pytest.mark.asyncio
async def test_prerun_inserts_running_row(db_sync):
"""Fire a no-op task; assert task_run row inserted with running status."""
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_999) # nonexistent image_id → no-op
rows = db_sync.execute(
select(TaskRun.queue, TaskRun.target_id, TaskRun.task_name)
.where(TaskRun.target_id == 999_999)
).all()
assert len(rows) == 1
queue, target_id, task_name = rows[0]
assert queue == "thumbnail"
assert target_id == 999_999
assert task_name.endswith(".generate_thumbnail")
@pytest.mark.asyncio
async def test_postrun_marks_ok_with_duration(db_sync):
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_998)
row = db_sync.execute(
select(
TaskRun.status, TaskRun.finished_at, TaskRun.duration_ms,
).where(TaskRun.target_id == 999_998)
).one()
status, finished_at, duration_ms = row
assert status == "ok"
assert finished_at is not None
assert duration_ms is not None and duration_ms >= 0
@pytest.mark.asyncio
async def test_failure_records_error_type_and_message(db_sync, monkeypatch):
"""Force a tracked task to raise; assert error fields populated."""
from backend.app.tasks.thumbnail import generate_thumbnail
def _explode(*a, **k):
raise ValueError("synthetic failure")
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run",
_explode,
)
with pytest.raises(ValueError):
generate_thumbnail.delay(999_997).get(propagate=True)
row = db_sync.execute(
select(
TaskRun.status, TaskRun.error_type, TaskRun.error_message,
TaskRun.finished_at,
).where(TaskRun.target_id == 999_997)
.order_by(TaskRun.id.desc()).limit(1)
).one()
status, error_type, error_message, finished_at = row
assert status == "error"
assert error_type == "ValueError"
assert "synthetic failure" in (error_message or "")
assert finished_at is not None
@pytest.mark.asyncio
async def test_soft_time_limit_records_timeout_status(db_sync, monkeypatch):
from celery.exceptions import SoftTimeLimitExceeded
from backend.app.tasks.thumbnail import generate_thumbnail
def _timeout(*a, **k):
raise SoftTimeLimitExceeded("simulated soft limit")
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run",
_timeout,
)
with pytest.raises(SoftTimeLimitExceeded):
generate_thumbnail.delay(999_996).get(propagate=True)
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.target_id == 999_996)
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert status == "timeout"
@pytest.mark.asyncio
async def test_untracked_celery_internal_tasks_skip(db_sync):
"""celery.chord_unlock is in the untracked list; firing it should
NOT insert a task_run row."""
before = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
# Send a chord_unlock-style task via apply (no real task to dispatch).
# The signal handler checks task.name against _UNTRACKED_TASK_NAMES;
# if name is in the list, no row is inserted. We verify by firing
# a regular task and asserting only ONE row appears, not two.
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_995)
after = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
assert after - before == 1
@pytest.mark.asyncio
async def test_args_summary_truncated_to_255(db_sync, monkeypatch):
"""Task with a very large repr(args); assert args_summary <= 255."""
from backend.app.tasks.thumbnail import generate_thumbnail
# Force a giant arg by patching repr(args) target — actually just
# call with a large int that produces a long repr.
big_id = int("9" * 200) # 200-char repr
# The signal handler reads repr(args), so this naturally produces
# a long args_summary that should get truncated to 255.
try:
generate_thumbnail.delay(big_id).get(timeout=5, propagate=False)
except Exception:
pass
args_summary = db_sync.execute(
select(TaskRun.args_summary)
.where(TaskRun.task_name.endswith(".generate_thumbnail"))
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert args_summary is None or len(args_summary) <= 255
@pytest.mark.asyncio
async def test_error_message_truncated_to_2000(db_sync, monkeypatch):
from backend.app.tasks.thumbnail import generate_thumbnail
huge_message = "X" * 5000
def _raise_huge(*a, **k):
raise RuntimeError(huge_message)
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run", _raise_huge,
)
with pytest.raises(RuntimeError):
generate_thumbnail.delay(999_994).get(propagate=True)
error_message = db_sync.execute(
select(TaskRun.error_message).where(TaskRun.target_id == 999_994)
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert error_message is not None
assert len(error_message) <= 2000
@pytest.mark.asyncio
async def test_signal_handler_failure_does_not_break_task(db_sync, monkeypatch):
"""Monkeypatch sync_session_factory in the signals module to raise;
assert the task still returns its result and the worker doesn't
propagate the signal-handler exception."""
from backend.app.tasks.thumbnail import generate_thumbnail
def _broken_factory():
raise RuntimeError("db down")
monkeypatch.setattr(
"backend.app.celery_signals.sync_session_factory", _broken_factory,
)
# Task should still execute (no-op on missing image) without raising.
result = generate_thumbnail.delay(999_993).get(timeout=5)
assert result is not None # task ran; signal handler errors swallowed
@pytest.mark.parametrize("args,expected", [
((42,), 42),
((42, "ignored"), 42),
(("123",), 123),
(("not-int",), None),
((), None),
(None, None),
])
def test_target_id_extracted_from_first_int_arg(args, expected):
"""Unit test the helper directly; no DB needed."""
from backend.app.celery_signals import _target_id_from_args
assert _target_id_from_args(args) == expected
def test_finalize_sets_retry_status_with_retry_count(db_sync):
"""Direct unit-shape test of _finalize's retry path. We can't
reliably trigger Celery's task_retry signal under eager mode with
monkeypatched .run (the bind=True 'self' injection breaks), so
test the finalize helper directly — that's the code the retry
signal handler delegates to anyway."""
from datetime import UTC, datetime
from backend.app.celery_signals import _finalize
from backend.app.models import TaskRun
# Seed a 'running' row that _finalize can target.
row = TaskRun(
celery_task_id="retry-test-tid",
queue="ml",
task_name="backend.app.tasks.fake.t",
target_id=999_992,
started_at=datetime.now(UTC),
status="running",
)
db_sync.add(row)
db_sync.commit()
_finalize(
"retry-test-tid", status="retry",
error_type="RuntimeError",
error_message="first attempt",
retry_count=1,
)
# Column-select assertions — _finalize mutated the row via its own
# session; re-read fresh state through db_sync.
db_sync.expire_all()
status, retry_count, error_type = db_sync.execute(
select(TaskRun.status, TaskRun.retry_count, TaskRun.error_type)
.where(TaskRun.celery_task_id == "retry-test-tid")
).one()
assert status == "retry"
assert retry_count == 1
assert error_type == "RuntimeError"
@pytest.mark.asyncio
async def test_worker_hostname_recorded(db_sync):
"""Sender's hostname should land in worker_hostname."""
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_991)
hostname = db_sync.execute(
select(TaskRun.worker_hostname).where(TaskRun.target_id == 999_991)
).scalar_one()
# Under eager mode hostname may be None or "celery@local"; just
# assert the column is touched (either string or None — never throws).
assert hostname is None or isinstance(hostname, str)