Compare commits

...

17 Commits

Author SHA1 Message Date
bvandeusen d1c0b82a22 Merge pull request 'v26.05.24.3: FC-3i System Activity dashboard + migration backup-gate retired + modal Escape' (#9) from dev into main 2026-05-24 21:47:53 -04:00
bvandeusen 37fcc74954 fix(fc3i): single-line celery.signals import + INT32 bounds on target_id + dict.fromkeys + rewrite retry test as direct _finalize unit
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:22:42 -04:00
bvandeusen 6a532c1497 fc3i(ui): Settings → Activity tab + Overview summary card
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:10:14 -04:00
bvandeusen 368613068a fc3i(ui): SystemActivityTab.vue — queues + failures + all-activity panes
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:09:20 -04:00
bvandeusen ddbb84d8aa fc3i(ui): SystemActivitySummary.vue — Overview-tab quick summary card (5s poll)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:08:30 -04:00
bvandeusen 9b252948f9 fc3i(ui): QueuesTable.vue — shared queue/worker/recent table (compact + detailed)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:57 -04:00
bvandeusen 89d0cb2124 fc3i(ui): systemActivity Pinia store — queues, workers, runs, failures
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:28 -04:00
bvandeusen 36611cbe00 fc3i: tests for recover_stalled_task_runs + prune_task_runs
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:07:01 -04:00
bvandeusen 48ef22445a fc3i: integration tests for /api/system/activity/* endpoints
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:06:20 -04:00
bvandeusen e523d0ac94 fc3i: integration tests for Celery signal → task_run lifecycle
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:05:32 -04:00
bvandeusen 541e2bfe6a fc3i: /api/system/activity blueprint — queues, workers, runs, failures
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:04:34 -04:00
bvandeusen 7782672a51 fc3i: recover_stalled_task_runs + prune_task_runs maintenance tasks + Beat entries
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:03:33 -04:00
bvandeusen d12b51f6b7 fc3i: Celery signal handlers populate task_run on every task lifecycle event
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:01:47 -04:00
bvandeusen 0cda46fcdb fc3i: alembic 0016 — task_run table with single + composite indexes
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:00:46 -04:00
bvandeusen 79fee98db4 fc3i: TaskRun model — per-Celery-task lifecycle audit row
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 21:00:14 -04:00
bvandeusen b1d68929c5 fix(tests): drop test_post_apply_without_backup_rejected (gate retired in 5535677)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 20:15:10 -04:00
bvandeusen 553567738e fix: drop migration backup-gate (FC-3h supersedes) + modal Escape via document-level listener so video focus can't swallow it
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 14:38:33 -04:00
19 changed files with 1936 additions and 34 deletions
+86
View File
@@ -0,0 +1,86 @@
"""fc3i: task_run table
Revision ID: 0016
Revises: 0015
Create Date: 2026-05-24
Additive only. New table records every Celery task attempt via signal
handlers (backend.app.celery_signals). Status is plain String(16) not
Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard-
fail at INSERT, String columns extend cleanly).
Composite indexes anticipate the three dashboard panes:
- (queue, started_at desc) — per-lane recent activity
- (status, started_at desc) — recent failures pane
- (task_name, started_at desc) — drill-down by task
Indexed columns get individual indexes via `index=True` on the model;
the composites below cover the multi-column lookups.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0016"
down_revision: Union[str, None] = "0015"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"task_run",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("celery_task_id", sa.String(length=64), nullable=False),
sa.Column("queue", sa.String(length=32), nullable=False),
sa.Column("task_name", sa.String(length=128), nullable=False),
sa.Column("target_id", sa.Integer(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=True),
sa.Column(
"status", sa.String(length=16), nullable=False,
server_default="running",
),
sa.Column("error_type", sa.String(length=128), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("retry_count", sa.Integer(), nullable=True),
sa.Column("worker_hostname", sa.String(length=128), nullable=True),
sa.Column("args_summary", sa.String(length=255), nullable=True),
)
# Single-column indexes (matches Mapped[...].index=True on model).
op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"])
op.create_index("ix_task_run_queue", "task_run", ["queue"])
op.create_index("ix_task_run_task_name", "task_run", ["task_name"])
op.create_index("ix_task_run_started_at", "task_run", ["started_at"])
op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"])
op.create_index("ix_task_run_status", "task_run", ["status"])
# Composite indexes for dashboard query patterns.
op.create_index(
"ix_task_run_queue_started",
"task_run", ["queue", sa.text("started_at DESC")],
)
op.create_index(
"ix_task_run_status_started",
"task_run", ["status", sa.text("started_at DESC")],
)
op.create_index(
"ix_task_run_name_started",
"task_run", ["task_name", sa.text("started_at DESC")],
)
def downgrade() -> None:
op.drop_index("ix_task_run_name_started", table_name="task_run")
op.drop_index("ix_task_run_status_started", table_name="task_run")
op.drop_index("ix_task_run_queue_started", table_name="task_run")
op.drop_index("ix_task_run_status", table_name="task_run")
op.drop_index("ix_task_run_finished_at", table_name="task_run")
op.drop_index("ix_task_run_started_at", table_name="task_run")
op.drop_index("ix_task_run_task_name", table_name="task_run")
op.drop_index("ix_task_run_queue", table_name="task_run")
op.drop_index("ix_task_run_celery_task_id", table_name="task_run")
op.drop_table("task_run")
+2
View File
@@ -33,6 +33,7 @@ def all_blueprints() -> list[Blueprint]:
from .showcase import showcase_bp
from .sources import sources_bp
from .suggestions import suggestions_bp
from .system_activity import system_activity_bp
from .tags import tags_bp
return [
api_bp,
@@ -44,6 +45,7 @@ def all_blueprints() -> list[Blueprint]:
artists_bp,
showcase_bp,
settings_bp,
system_activity_bp,
import_admin_bp,
migrate_bp,
suggestions_bp,
+6 -1
View File
@@ -24,7 +24,12 @@ _VALID_KINDS = frozenset({
"ml_queue", "verify", "rollback", "cleanup",
})
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
# Backup gate retired 2026-05-24 — operator-flagged the speculative-safety
# requirement was actively blocking the UI ingest path (backup itself is
# unreliable on large NFS-backed image libraries) and FC-3h will rewrite
# the backup surface as a first-class feature with its own scheduling
# + recovery. Leaving the constant for historical grep.
_APPLY_KINDS: frozenset[str] = frozenset()
def _bad(error: str, *, status: int = 400, **extra):
+209
View File
@@ -0,0 +1,209 @@
"""FC-3i: system activity dashboard endpoints.
Read-only. Combines Redis-broker queue depths (LLEN per queue),
Celery worker introspection (celery inspect), and the task_run DB
history into the surfaces the SystemActivityTab UI consumes.
All filesystem/sync-client work goes through asyncio.to_thread per
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
"""
from __future__ import annotations
import asyncio
import time
from datetime import UTC, datetime, timedelta
from quart import Blueprint, jsonify, request
from sqlalchemy import desc, func, select
from ..config import get_config
from ..extensions import get_session
from ..models import TaskRun
system_activity_bp = Blueprint(
"system_activity", __name__, url_prefix="/api/system/activity",
)
# Canonical queue order — must match celery_app.task_routes. UI renders
# in this order; queues with no LLEN response show as null rather than
# absent.
_QUEUE_NAMES = (
"default", "import", "thumbnail", "ml",
"download", "scan", "maintenance",
)
# Cache module-level so all requests share the cache between polls.
# Tests can reset via direct dict mutation if needed.
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
_QUEUE_CACHE_TTL = 2.0
_WORKER_CACHE_TTL = 5.0
def _read_queues_sync() -> dict:
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
asyncio.to_thread. Per-queue try/except returns None on failure so
one bad queue doesn't break the whole response."""
import redis # local import; only this endpoint needs it
cfg = get_config()
client = redis.Redis.from_url(cfg.celery_broker_url)
out: dict = {}
for name in _QUEUE_NAMES:
try:
out[name] = int(client.llen(name))
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
out[name] = None
return {
"queues": out,
"fetched_at": datetime.now(UTC).isoformat(),
}
def _read_workers_sync() -> dict:
"""celery inspect active_queues + active. Returns per-worker info."""
from ..celery_app import celery as celery_app
insp = celery_app.control.inspect(timeout=2.0)
active_queues = insp.active_queues() or {}
active_tasks = insp.active() or {}
workers: dict = {}
for hostname, queues in active_queues.items():
workers[hostname] = {
"queues": sorted({q["name"] for q in queues}),
"active_count": len(active_tasks.get(hostname, [])),
}
return {
"workers": workers,
"fetched_at": datetime.now(UTC).isoformat(),
}
@system_activity_bp.route("/queues", methods=["GET"])
async def get_queues():
"""Per-queue Redis LLEN. Cached 2s.
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
"""
now = time.time()
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
_QUEUE_CACHE["ts"] = now
return jsonify(_QUEUE_CACHE["data"])
@system_activity_bp.route("/workers", methods=["GET"])
async def get_workers():
"""Live celery inspect. Cached 5s.
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
"""
now = time.time()
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
_WORKER_CACHE["ts"] = now
return jsonify(_WORKER_CACHE["data"])
@system_activity_bp.route("/runs", methods=["GET"])
async def list_runs():
"""Paginated task_run history. Query params:
queue=<name> filter to one queue
status=<status> filter to one status (running/ok/error/timeout/retry)
limit=<int> default 50, max 200
before_id=<int> cursor for keyset pagination
Response: {runs: [...], next_cursor: id|null}
"""
try:
limit = min(int(request.args.get("limit", "50")), 200)
except ValueError:
return jsonify({"error": "invalid_limit"}), 400
if limit < 1:
return jsonify({"error": "invalid_limit"}), 400
queue = request.args.get("queue")
status = request.args.get("status")
before_id_raw = request.args.get("before_id")
before_id = int(before_id_raw) if before_id_raw else None
async with get_session() as session:
stmt = select(TaskRun).order_by(desc(TaskRun.id))
if queue:
stmt = stmt.where(TaskRun.queue == queue)
if status:
stmt = stmt.where(TaskRun.status == status)
if before_id is not None:
stmt = stmt.where(TaskRun.id < before_id)
stmt = stmt.limit(limit + 1)
rows = (await session.execute(stmt)).scalars().all()
has_more = len(rows) > limit
rows = rows[:limit]
return jsonify({
"runs": [_row_to_dict(r) for r in rows],
"next_cursor": rows[-1].id if has_more and rows else None,
})
@system_activity_bp.route("/failures", methods=["GET"])
async def list_failures():
"""Recent failures across all lanes (24h window).
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
"""
try:
limit = min(int(request.args.get("limit", "50")), 200)
except ValueError:
return jsonify({"error": "invalid_limit"}), 400
since = datetime.now(UTC) - timedelta(hours=24)
async with get_session() as session:
recent_stmt = (
select(TaskRun)
.where(TaskRun.status.in_(["error", "timeout"]))
.where(TaskRun.finished_at >= since)
.order_by(desc(TaskRun.finished_at))
.limit(limit)
)
recent = (await session.execute(recent_stmt)).scalars().all()
count_stmt = (
select(TaskRun.error_type, func.count(TaskRun.id))
.where(TaskRun.status.in_(["error", "timeout"]))
.where(TaskRun.finished_at >= since)
.group_by(TaskRun.error_type)
.order_by(desc(func.count(TaskRun.id)))
)
counts = (await session.execute(count_stmt)).all()
return jsonify({
"recent": [_row_to_dict(r) for r in recent],
"count_by_type": {
(row[0] or "Unknown"): row[1]
for row in counts
},
"since": since.isoformat(),
})
def _row_to_dict(r: TaskRun) -> dict:
return {
"id": r.id,
"queue": r.queue,
"task_name": r.task_name,
"target_id": r.target_id,
"celery_task_id": r.celery_task_id,
"started_at": r.started_at.isoformat() if r.started_at else None,
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
"duration_ms": r.duration_ms,
"status": r.status,
"error_type": r.error_type,
"error_message": r.error_message,
"retry_count": r.retry_count,
"worker_hostname": r.worker_hostname,
"args_summary": r.args_summary,
}
+11
View File
@@ -81,9 +81,20 @@ def make_celery() -> Celery:
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
"schedule": 86400.0, # daily
},
"recover-stalled-task-runs": {
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
},
"prune-task-runs": {
"task": "backend.app.tasks.maintenance.prune_task_runs",
"schedule": 86400.0, # daily
},
},
timezone="UTC",
)
# FC-3i: register task_run signal handlers (side-effect import).
from . import celery_signals # noqa: F401
return app
+210
View File
@@ -0,0 +1,210 @@
"""FC-3i: task_run lifecycle via Celery signals.
Subscribes to task_prerun / task_postrun / task_failure / task_retry
and persists one task_run row per task attempt. Drop-in for every
existing and future Celery task — no per-task instrumentation.
Signal handlers run inside the worker process (sync context); DB
writes go through the existing shared sync engine
(backend.app.tasks._sync_engine.sync_session_factory) — one engine
per worker process, not per-task, so we don't blow Postgres
max_connections under load (the reason FC-3g shared-engine fix
existed).
Failure-mode discipline (operator-pressed point): every handler is
wrapped in try/except that swallows + logs. If the DB is down or the
handler has a bug, the real task still runs — the dashboard goes
dark for that interval. Monitoring NEVER breaks the thing it's
monitoring.
"""
import logging
from datetime import UTC, datetime
from celery.exceptions import SoftTimeLimitExceeded
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
from .models import TaskRun
from .tasks._sync_engine import sync_session_factory
log = logging.getLogger(__name__)
# Celery-internal tasks that would generate dashboard noise without
# operational value. Conservative list; extend only when a specific
# task proves noisy.
_UNTRACKED_TASK_NAMES = frozenset({
"celery.chord_unlock",
"celery.backend_cleanup",
"celery.chunks",
})
_MAX_ERROR_MESSAGE_LEN = 2000
_MAX_ARGS_SUMMARY_LEN = 255
_MAX_WORKER_HOSTNAME_LEN = 128
# PostgreSQL Integer is signed 32-bit. Tasks called with a first-arg
# int outside this range (e.g. an absurdly large mock value, or a
# string-of-digits coercible to int but bigger than 2^31-1) would crash
# the INSERT with NumericValueOutOfRange. Bound the recorded value to
# the column's range; values outside become None (target_id is
# nullable, so this is safe).
_INT32_MAX = 2_147_483_647
_INT32_MIN = -2_147_483_648
def _queue_for(task) -> str:
"""Reverse the task→queue routing from celery_app.task_routes.
Keep in sync if task_routes is reordered."""
name = getattr(task, "name", "") or ""
if name.startswith("backend.app.tasks.import_file."):
return "import"
if name.startswith("backend.app.tasks.ml."):
return "ml"
if name.startswith("backend.app.tasks.thumbnail."):
return "thumbnail"
if name.startswith("backend.app.tasks.download."):
return "download"
if name.startswith("backend.app.tasks.scan."):
return "scan"
if name.startswith((
"backend.app.tasks.maintenance.",
"backend.app.tasks.migration.",
)):
return "maintenance"
return "default"
def _target_id_from_args(args) -> int | None:
"""Best-effort: if the first positional arg parses as int AND fits
in the column's signed-32-bit range, record it as target_id
(image_id, source_id, etc.). Never raises."""
if not args:
return None
try:
value = int(args[0])
except (TypeError, ValueError):
return None
if value < _INT32_MIN or value > _INT32_MAX:
return None
return value
def _truncate(s, limit: int) -> str | None:
if s is None:
return None
text = str(s)
return text if len(text) <= limit else text[:limit]
def _is_tracked(task_name: str | None) -> bool:
return bool(task_name) and task_name not in _UNTRACKED_TASK_NAMES
@task_prerun.connect
def _on_prerun(sender=None, task_id=None, task=None, args=None,
kwargs=None, **_):
if not _is_tracked(getattr(task, "name", None)):
return
try:
Session = sync_session_factory()
with Session() as session:
session.add(TaskRun(
celery_task_id=task_id or "",
queue=_queue_for(task),
task_name=task.name,
target_id=_target_id_from_args(args),
started_at=datetime.now(UTC),
status="running",
args_summary=_truncate(repr(args), _MAX_ARGS_SUMMARY_LEN),
worker_hostname=_truncate(
getattr(sender, "hostname", None),
_MAX_WORKER_HOSTNAME_LEN,
),
))
session.commit()
except Exception: # noqa: BLE001 — never break the worker
log.exception("task_run prerun insert failed (task=%s)",
getattr(task, "name", "?"))
def _finalize(task_id: str, *, status: str,
error_type: str | None = None,
error_message: str | None = None,
retry_count: int | None = None) -> None:
"""Shared write path for postrun/failure/retry. Picks the most-
recent task_run row for this celery_task_id that's still 'running'
(retries reuse the same celery_task_id; each new attempt's prerun
inserts a fresh row, so finalize targets the latest running row)."""
try:
from sqlalchemy import select
Session = sync_session_factory()
now = datetime.now(UTC)
with Session() as session:
row = session.execute(
select(TaskRun)
.where(TaskRun.celery_task_id == task_id)
.where(TaskRun.status == "running")
.order_by(TaskRun.id.desc())
.limit(1)
).scalar_one_or_none()
if row is None:
return # no prerun row (untracked, insert failed, or already finalized)
row.finished_at = now
row.duration_ms = int(
(now - row.started_at).total_seconds() * 1000
)
row.status = status
if error_type is not None:
row.error_type = _truncate(error_type, 128)
if error_message is not None:
row.error_message = _truncate(error_message, _MAX_ERROR_MESSAGE_LEN)
if retry_count is not None:
row.retry_count = retry_count
session.commit()
except Exception: # noqa: BLE001
log.exception("task_run finalize failed (task_id=%s)", task_id)
@task_postrun.connect
def _on_postrun(sender=None, task_id=None, task=None, args=None,
kwargs=None, retval=None, state=None, **_):
if not _is_tracked(getattr(task, "name", None)):
return
# state is one of SUCCESS/FAILURE/RETRY/etc. Only handle SUCCESS;
# task_failure handles FAILURE explicitly (with the exception).
if state != "SUCCESS":
return
_finalize(task_id, status="ok")
@task_failure.connect
def _on_failure(sender=None, task_id=None, exception=None,
args=None, kwargs=None, einfo=None, **_):
if not _is_tracked(getattr(sender, "name", None)):
return
status = ("timeout"
if isinstance(exception, SoftTimeLimitExceeded)
else "error")
_finalize(
task_id, status=status,
error_type=type(exception).__name__ if exception else "Unknown",
error_message=str(exception) if exception else None,
)
@task_retry.connect
def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
if not _is_tracked(getattr(sender, "name", None)):
return
task_id = getattr(request, "id", None)
if not task_id:
return
# Mark current attempt's row as 'retry' (terminal for this row).
# The next attempt's task_prerun inserts a fresh row.
_finalize(
task_id, status="retry",
error_type=type(reason).__name__ if reason else "Retry",
error_message=str(reason) if reason else None,
retry_count=getattr(request, "retries", 0),
)
+2
View File
@@ -21,6 +21,7 @@ from .tag_alias import TagAlias
from .tag_allowlist import TagAllowlist
from .tag_reference_embedding import TagReferenceEmbedding
from .tag_suggestion_rejection import TagSuggestionRejection
from .task_run import TaskRun
__all__ = [
"Base",
@@ -46,4 +47,5 @@ __all__ = [
"TagAllowlist",
"TagReferenceEmbedding",
"TagSuggestionRejection",
"TaskRun",
]
+48
View File
@@ -0,0 +1,48 @@
"""FC-3i: task_run — per-Celery-task lifecycle audit row.
One row inserted by the task_prerun signal at task start, updated by
task_postrun / task_failure / task_retry. The shape supports the
SystemActivity dashboard's three panes: per-queue queue+worker summary,
recent failures (24h, grouped by error_type), and full paginated
activity history.
Retention: ok rows pruned after 24h, error/timeout after 7d (see
backend.app.tasks.maintenance.prune_task_runs).
Recovery: rows stuck in 'running' for >5 min flipped to 'error' by
backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
"""
from datetime import datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class TaskRun(Base):
__tablename__ = "task_run"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
celery_task_id: Mapped[str] = mapped_column(
String(64), nullable=False, index=True,
)
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True,
)
finished_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True,
)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
status: Mapped[str] = mapped_column(
String(16), nullable=False, default="running", index=True,
)
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
retry_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
worker_hostname: Mapped[str | None] = mapped_column(String(128), nullable=True)
args_summary: Mapped[str | None] = mapped_column(String(255), nullable=True)
+68 -1
View File
@@ -9,7 +9,7 @@ from PIL import Image
from sqlalchemy import delete, select, update
from ..celery_app import celery
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
from ..utils.phash import compute_phash
from ._sync_engine import sync_session_factory as _sync_session_factory
@@ -20,6 +20,8 @@ OLD_TASK_DAYS = 7
PHASH_PAGE = 500
VERIFY_PAGE = 200
FFPROBE_TIMEOUT_SECONDS = 10
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
@@ -80,6 +82,71 @@ def cleanup_old_tasks() -> int:
return result.rowcount or 0
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
def recover_stalled_task_runs() -> int:
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
to 'error'. FC-3i.
A row gets stuck when the worker dies without emitting
task_postrun / task_failure (e.g. OOM, container restart between
signals, signal handler raised+logged). Shares the 5-min threshold
with recover_interrupted_tasks for consistency.
"""
SessionLocal = _sync_session_factory()
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
with SessionLocal() as session:
result = session.execute(
update(TaskRun)
.where(TaskRun.status == "running")
.where(TaskRun.started_at < cutoff)
.values(
status="error",
error_type="RecoverySweep",
error_message=(
f"no completion signal received within "
f"{STUCK_THRESHOLD_MINUTES} min"
),
finished_at=datetime.now(UTC),
)
)
session.commit()
return result.rowcount or 0
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
def prune_task_runs() -> dict:
"""Daily retention for task_run rows. FC-3i.
- 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
Success is high-volume, not interesting after a day.
- 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
(7 days default). Failures are operationally interesting longer.
- 'running' rows: NEVER deleted by this task. The recovery sweep
(recover_stalled_task_runs) is the mechanism that flips them to
terminal state; prune doesn't touch in-flight state.
- 'retry' rows: treated as failures (>7d).
Returns dict of how many rows were deleted in each bucket.
"""
SessionLocal = _sync_session_factory()
now = datetime.now(UTC)
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
with SessionLocal() as session:
ok_deleted = session.execute(
delete(TaskRun)
.where(TaskRun.status == "ok")
.where(TaskRun.finished_at < ok_cutoff)
).rowcount or 0
fail_deleted = session.execute(
delete(TaskRun)
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
.where(TaskRun.finished_at < fail_cutoff)
).rowcount or 0
session.commit()
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
def backfill_phash() -> int:
"""Recompute phash for stored images that have none (imported before
+26 -14
View File
@@ -2,9 +2,6 @@
<Teleport to="body">
<div
class="fc-viewer" role="dialog" aria-modal="true"
@keydown.esc="$emit('close')"
@keydown.left="onArrowLeft"
@keydown.right="onArrowRight"
tabindex="-1" ref="rootEl"
>
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
@@ -68,7 +65,7 @@ import VideoCanvas from './VideoCanvas.vue'
import TagPanel from './TagPanel.vue'
import ProvenancePanel from './ProvenancePanel.vue'
defineEmits(['close'])
const emit = defineEmits(['close'])
const modal = useModalStore()
const rootEl = ref(null)
@@ -86,14 +83,39 @@ const integrityBadge = computed(() => {
})
let prevBodyOverflow = null
// Document-level keyboard handler. Bound on document (not on the modal
// root) because <video> elements capture focus when the user interacts
// with their controls — when video has focus, Escape and arrow keys
// don't bubble reliably to ancestors. Listening on document side-steps
// that. Filtered via isTextEntry so tag/comment inputs still get their
// own keystrokes.
function onKeyDown(ev) {
if (ev.key === 'Escape') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
emit('close')
} else if (ev.key === 'ArrowLeft') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goPrev()
} else if (ev.key === 'ArrowRight') {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goNext()
}
}
onMounted(async () => {
prevBodyOverflow = document.body.style.overflow
document.body.style.overflow = 'hidden'
document.addEventListener('keydown', onKeyDown, true)
await nextFrame()
rootEl.value?.focus()
})
onUnmounted(() => {
document.body.style.overflow = prevBodyOverflow
document.removeEventListener('keydown', onKeyDown, true)
})
watch(() => modal.currentImageId, async () => {
@@ -105,16 +127,6 @@ function nextFrame() {
return new Promise(resolve => requestAnimationFrame(resolve))
}
function onArrowLeft(ev) {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goPrev()
}
function onArrowRight(ev) {
if (isTextEntry(ev.target)) return
ev.preventDefault()
modal.goNext()
}
function isTextEntry(el) {
if (!el) return false
const tag = el.tagName
@@ -0,0 +1,109 @@
<template>
<v-table density="compact" class="fc-queues-table">
<thead>
<tr>
<th>Queue</th>
<th class="text-right">Depth</th>
<th class="text-right">Workers</th>
<th v-if="!compact" class="text-right">Active</th>
<th class="text-right">Last min: ok / err</th>
</tr>
</thead>
<tbody>
<tr v-for="q in QUEUE_NAMES" :key="q">
<td>{{ q }}</td>
<td class="text-right fc-tabular" :class="depthClass(q)">
{{ formatDepth(q) }}
</td>
<td class="text-right fc-tabular">{{ workerCount(q) }}</td>
<td v-if="!compact" class="text-right fc-tabular">{{ activeCount(q) }}</td>
<td class="text-right fc-tabular">
<span class="fc-ok">{{ recentOk(q) }}</span>
<span class="fc-sep">/</span>
<span :class="recentErr(q) > 0 ? 'fc-err' : 'fc-muted'">
{{ recentErr(q) }}<span v-if="recentErr(q) > 0"> </span>
</span>
</td>
</tr>
</tbody>
</v-table>
</template>
<script setup>
import { computed } from 'vue'
const props = defineProps({
queues: { type: Object, default: null }, // store.queues
workers: { type: Object, default: null }, // store.workers
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
compact: { type: Boolean, default: false },
})
const QUEUE_NAMES = [
'default', 'import', 'thumbnail', 'ml',
'download', 'scan', 'maintenance',
]
function formatDepth(name) {
const d = props.queues?.queues?.[name]
if (d == null) return '—'
return d.toLocaleString()
}
function depthClass(name) {
const d = props.queues?.queues?.[name]
const workers = workerCount(name)
// Queue has depth but no workers → warning.
if (d != null && d > 0 && workers === 0) return 'fc-warn'
return ''
}
function workerCount(name) {
if (!props.workers?.workers) return 0
let count = 0
for (const info of Object.values(props.workers.workers)) {
if (info.queues?.includes(name)) count++
}
return count
}
function activeCount(name) {
if (!props.workers?.workers) return 0
// Active count is per-worker, not per-queue. Approximate: sum of
// active_count across workers that subscribe to this queue.
let count = 0
for (const info of Object.values(props.workers.workers)) {
if (info.queues?.includes(name)) count += info.active_count || 0
}
return count
}
const recentByQueue = computed(() => {
const out = {}
for (const r of props.recentMinute) {
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
if (r.status === 'ok') out[r.queue].ok++
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
}
return out
})
function recentOk(name) { return recentByQueue.value[name]?.ok || 0 }
function recentErr(name) { return recentByQueue.value[name]?.err || 0 }
</script>
<style scoped>
.fc-queues-table { background: transparent; }
.fc-tabular {
font-variant-numeric: tabular-nums;
font-feature-settings: 'tnum';
}
.fc-ok { color: rgb(var(--v-theme-success, 76 175 80)); }
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
.fc-warn { color: rgb(var(--v-theme-warning, 255 179 0)); font-weight: 500; }
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
.fc-sep {
margin: 0 4px;
color: rgb(var(--v-theme-on-surface-variant));
}
</style>
@@ -0,0 +1,64 @@
<template>
<v-card class="fc-activity-summary">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-pulse" size="small" />
<span>System activity (last min)</span>
<v-spacer />
<v-btn
variant="text" size="small" rounded="pill"
@click="$emit('open-activity')"
>
View activity
<v-icon end size="small">mdi-arrow-right</v-icon>
</v-btn>
</v-card-title>
<v-card-text>
<v-alert
v-if="store.lastError"
type="warning" variant="tonal" density="compact" class="mb-3"
>
{{ store.lastError }}
</v-alert>
<QueuesTable
:queues="store.queues"
:workers="store.workers"
:recent-minute="store.recentMinute"
compact
/>
</v-card-text>
</v-card>
</template>
<script setup>
import { onMounted, onUnmounted } from 'vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
import QueuesTable from './QueuesTable.vue'
defineEmits(['open-activity'])
const store = useSystemActivityStore()
let pollId = null
function pollOnce() {
if (document.hidden) return
store.loadQueues()
store.loadWorkers()
store.loadRecentMinute()
}
onMounted(() => {
pollOnce()
pollId = setInterval(pollOnce, 5000)
})
onUnmounted(() => {
if (pollId) { clearInterval(pollId); pollId = null }
})
</script>
<style scoped>
.fc-activity-summary { border-radius: 8px; }
</style>
@@ -0,0 +1,280 @@
<template>
<div class="fc-activity">
<!-- Queues + workers pane -->
<v-card class="mb-4">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-pulse" size="small" />
<span>Queues + workers</span>
<v-spacer />
<span class="text-caption fc-muted">
updated {{ formatRelative(store.queues?.fetched_at) }}
</span>
</v-card-title>
<v-card-text>
<QueuesTable
:queues="store.queues"
:workers="store.workers"
:recent-minute="store.recentMinute"
/>
</v-card-text>
</v-card>
<!-- Recent failures pane -->
<v-card class="mb-4">
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-alert-circle-outline" size="small" />
<span>Recent failures (last 24h)</span>
<v-spacer />
<span class="text-caption fc-muted">
{{ failureCount }} failures
</span>
</v-card-title>
<v-card-text>
<div v-if="errorTypes.length" class="mb-3 fc-pills">
<v-chip
v-for="t in errorTypes" :key="t.name"
:color="t.name === filterErrorType ? 'accent' : undefined"
size="small" variant="tonal" closable
@click="toggleErrorTypeFilter(t.name)"
@click:close="toggleErrorTypeFilter(t.name)"
>
{{ t.name }} × {{ t.count }}
</v-chip>
</div>
<v-table density="compact">
<thead>
<tr>
<th>Time</th>
<th>Queue</th>
<th>Task</th>
<th>Target</th>
<th>Error</th>
</tr>
</thead>
<tbody>
<tr v-for="r in filteredFailures" :key="r.id">
<td class="fc-tabular" :title="r.finished_at">
{{ formatRelative(r.finished_at) }}
</td>
<td>{{ r.queue }}</td>
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
<td class="fc-err" :title="r.error_message">
{{ r.error_type }}
</td>
</tr>
<tr v-if="!filteredFailures.length">
<td colspan="5" class="text-center fc-muted py-4">
No failures in the last 24h.
</td>
</tr>
</tbody>
</v-table>
</v-card-text>
</v-card>
<!-- All activity pane -->
<v-card>
<v-card-title class="d-flex align-center" style="gap: 10px;">
<v-icon icon="mdi-format-list-bulleted" size="small" />
<span>All recent activity</span>
<v-spacer />
<v-select
v-model="filterQueue"
:items="queueOptions" density="compact" hide-details
style="max-width: 180px;"
@update:model-value="onFilterChange"
/>
<v-select
v-model="filterStatus"
:items="statusOptions" density="compact" hide-details
style="max-width: 180px;"
@update:model-value="onFilterChange"
/>
<v-btn
variant="text" size="small" rounded="pill"
@click="onRefresh"
>
<v-icon start size="small">mdi-refresh</v-icon>
Refresh
</v-btn>
</v-card-title>
<v-card-text>
<v-table density="compact">
<thead>
<tr>
<th></th>
<th>Time</th>
<th>Queue</th>
<th>Task</th>
<th>Target</th>
<th class="text-right">Duration</th>
<th>Status</th>
</tr>
</thead>
<tbody>
<tr v-for="r in store.runs" :key="r.id">
<td><v-icon size="small" :color="statusColor(r.status)">{{ statusIcon(r.status) }}</v-icon></td>
<td class="fc-tabular" :title="r.started_at">{{ formatRelative(r.started_at) }}</td>
<td>{{ r.queue }}</td>
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
<td class="text-right fc-tabular">{{ formatDuration(r.duration_ms) }}</td>
<td>{{ r.status }}</td>
</tr>
<tr v-if="!store.runs.length && !store.loading.runs">
<td colspan="7" class="text-center fc-muted py-4">
No activity yet.
</td>
</tr>
</tbody>
</v-table>
<div v-if="store.runsHasMore" class="d-flex justify-center py-3">
<v-btn
variant="text" size="small"
:loading="store.loading.runs"
@click="onLoadMore"
>Load more</v-btn>
</div>
</v-card-text>
</v-card>
</div>
</template>
<script setup>
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
import QueuesTable from './QueuesTable.vue'
const store = useSystemActivityStore()
const filterQueue = ref(null)
const filterStatus = ref(null)
const filterErrorType = ref(null)
const queueOptions = [
{ title: 'All queues', value: null },
{ title: 'import', value: 'import' },
{ title: 'thumbnail', value: 'thumbnail' },
{ title: 'ml', value: 'ml' },
{ title: 'download', value: 'download' },
{ title: 'scan', value: 'scan' },
{ title: 'maintenance', value: 'maintenance' },
{ title: 'default', value: 'default' },
]
const statusOptions = [
{ title: 'All statuses', value: null },
{ title: 'Running', value: 'running' },
{ title: 'OK', value: 'ok' },
{ title: 'Error', value: 'error' },
{ title: 'Timeout', value: 'timeout' },
{ title: 'Retry', value: 'retry' },
]
let queuesPollId = null
let failuresPollId = null
function pollQueues() {
if (document.hidden) return
store.loadQueues()
store.loadWorkers()
store.loadRecentMinute()
}
function pollFailures() {
if (document.hidden) return
store.loadFailures()
}
onMounted(() => {
pollQueues()
pollFailures()
store.setFilter({ queue: null, status: null })
store.loadRuns({ reset: true })
queuesPollId = setInterval(pollQueues, 3000)
failuresPollId = setInterval(pollFailures, 15000)
})
onUnmounted(() => {
if (queuesPollId) clearInterval(queuesPollId)
if (failuresPollId) clearInterval(failuresPollId)
})
const errorTypes = computed(() => {
const counts = store.failures?.count_by_type || {}
return Object.entries(counts).map(([name, count]) => ({ name, count }))
})
const failureCount = computed(() => (store.failures?.recent || []).length)
const filteredFailures = computed(() => {
const all = store.failures?.recent || []
if (!filterErrorType.value) return all
return all.filter(r => r.error_type === filterErrorType.value)
})
function toggleErrorTypeFilter(name) {
filterErrorType.value = filterErrorType.value === name ? null : name
if (filterErrorType.value) {
// Mirror into the All-activity table too.
filterStatus.value = 'error'
onFilterChange()
}
}
function onFilterChange() {
store.setFilter({ queue: filterQueue.value, status: filterStatus.value })
store.loadRuns({ reset: true })
}
function onLoadMore() { store.loadRuns({ reset: false }) }
function onRefresh() { store.loadRuns({ reset: true }) }
function statusIcon(status) {
return {
ok: 'mdi-check-circle',
error: 'mdi-close-circle',
timeout: 'mdi-clock-alert',
retry: 'mdi-refresh',
running: 'mdi-timer-sand',
}[status] || 'mdi-help-circle'
}
function statusColor(status) {
return {
ok: 'success', error: 'error', timeout: 'warning',
retry: 'info', running: 'accent',
}[status] || 'on-surface-variant'
}
function shortTaskName(name) {
if (!name) return ''
const parts = name.split('.')
return parts.slice(-1)[0]
}
function formatDuration(ms) {
if (ms == null) return '—'
if (ms < 1000) return `${ms} ms`
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
return `${(ms / 60_000).toFixed(1)} min`
}
function formatRelative(iso) {
if (!iso) return '—'
const then = new Date(iso).getTime()
const now = Date.now()
const diff = Math.max(0, (now - then) / 1000)
if (diff < 60) return `${Math.floor(diff)}s ago`
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
return `${Math.floor(diff / 86400)}d ago`
}
</script>
<style scoped>
.fc-activity { padding: 0; }
.fc-pills {
display: flex; gap: 6px; flex-wrap: wrap;
}
.fc-tabular {
font-variant-numeric: tabular-nums;
font-feature-settings: 'tnum';
}
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
</style>
+103
View File
@@ -0,0 +1,103 @@
import { defineStore } from 'pinia'
import { ref } from 'vue'
import { useApi } from '../composables/useApi.js'
export const useSystemActivityStore = defineStore('systemActivity', () => {
const api = useApi()
// Live polled state.
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
const recentMinute = ref([]) // last-60s rows (for Overview summary)
const failures = ref(null) // { recent, count_by_type, since }
// Paginated runs (Activity tab "All recent activity" pane).
const runs = ref([])
const runsCursor = ref(null)
const runsHasMore = ref(false)
const runsFilter = ref({ queue: null, status: null, limit: 50 })
const loading = ref({ queues: false, workers: false, runs: false, failures: false })
const lastError = ref(null)
async function loadQueues() {
loading.value.queues = true
lastError.value = null
try {
queues.value = await api.get('/api/system/activity/queues')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.queues = false
}
}
async function loadWorkers() {
loading.value.workers = true
lastError.value = null
try {
workers.value = await api.get('/api/system/activity/workers')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.workers = false
}
}
async function loadRecentMinute() {
// Used by the Overview summary card: pull last 60s of runs to compute
// per-queue ok/err counts. One call covers all queues; UI groups.
try {
const body = await api.get('/api/system/activity/runs', {
params: { limit: 200 },
})
recentMinute.value = body.runs || []
} catch (e) {
lastError.value = e.message
}
}
async function loadRuns({ reset = false } = {}) {
loading.value.runs = true
lastError.value = null
try {
const params = { limit: runsFilter.value.limit }
if (runsFilter.value.queue) params.queue = runsFilter.value.queue
if (runsFilter.value.status) params.status = runsFilter.value.status
if (!reset && runsCursor.value) params.before_id = runsCursor.value
const body = await api.get('/api/system/activity/runs', { params })
runs.value = reset ? body.runs : [...runs.value, ...body.runs]
runsCursor.value = body.next_cursor
runsHasMore.value = body.next_cursor !== null
} catch (e) {
lastError.value = e.message
} finally {
loading.value.runs = false
}
}
async function loadFailures() {
loading.value.failures = true
lastError.value = null
try {
failures.value = await api.get('/api/system/activity/failures')
} catch (e) {
lastError.value = e.message
} finally {
loading.value.failures = false
}
}
function setFilter(filter) {
runsFilter.value = { ...runsFilter.value, ...filter }
}
return {
queues, workers, recentMinute, failures,
runs, runsCursor, runsHasMore, runsFilter,
loading, lastError,
loadQueues, loadWorkers, loadRecentMinute,
loadRuns, loadFailures, setFilter,
}
})
+11
View File
@@ -2,6 +2,7 @@
<v-container fluid class="py-6">
<v-tabs v-model="tab" color="accent" class="mb-4">
<v-tab value="overview">Overview</v-tab>
<v-tab value="activity">Activity</v-tab>
<v-tab value="import">Import</v-tab>
<v-tab value="maintenance">Maintenance</v-tab>
</v-tabs>
@@ -9,6 +10,10 @@
<v-window v-model="tab">
<v-window-item value="overview">
<SystemStatsCards :stats="system.stats" />
<SystemActivitySummary
class="mt-4"
@open-activity="tab = 'activity'"
/>
<v-alert
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
type="info" variant="tonal" class="mt-4" closable
@@ -18,6 +23,10 @@
</v-alert>
</v-window-item>
<v-window-item value="activity">
<SystemActivityTab />
</v-window-item>
<v-window-item value="import">
<ImportTriggerPanel />
<v-divider class="my-6" />
@@ -38,6 +47,8 @@ import { onMounted, onUnmounted, ref, watch } from 'vue'
import { useSystemStore } from '../stores/system.js'
import { useImportStore } from '../stores/import.js'
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
import ImportTaskList from '../components/settings/ImportTaskList.vue'
+5 -18
View File
@@ -83,24 +83,11 @@ async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
assert resp.status_code == 202
@pytest.mark.asyncio
async def test_post_apply_without_backup_rejected(client, monkeypatch):
monkeypatch.setattr(
"backend.app.api.migrate._has_recent_pre_migration_backup",
lambda: False,
)
export = {
"source_app": "gallerysubscriber", "schema_version": 1,
"subscriptions": [], "credentials": [],
}
resp = await client.post(
"/api/migrate/gs_ingest",
form={"dry_run": "false"},
files={"export_file": _file_storage(export, "gs-export.json")},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "no_backup"
# Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
# the migration backup-gate rejected applies without a recent backup.
# That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
# FC-3h will own backup as a first-class feature) so the test was
# testing dead behavior. Removed rather than rewritten.
@pytest.mark.asyncio
+267
View File
@@ -0,0 +1,267 @@
"""FC-3i: /api/system/activity/* endpoint tests.
Mocks Redis LLEN + celery inspect via monkeypatch on the module-level
_read_queues_sync / _read_workers_sync helpers, so tests don't need
a live broker or running workers.
"""
from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from backend.app import create_app
from backend.app.api import system_activity as activity_module
from backend.app.models import TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture
async def app():
return create_app()
@pytest.fixture
async def client(app):
async with app.test_client() as c:
yield c
@pytest.fixture(autouse=True)
def _reset_caches(monkeypatch):
"""Clear the module-level caches between tests so cache state from
one test doesn't leak into the next."""
monkeypatch.setitem(activity_module._QUEUE_CACHE, "data", None)
monkeypatch.setitem(activity_module._QUEUE_CACHE, "ts", 0.0)
monkeypatch.setitem(activity_module._WORKER_CACHE, "data", None)
monkeypatch.setitem(activity_module._WORKER_CACHE, "ts", 0.0)
# --- /queues -------------------------------------------------------
@pytest.mark.asyncio
async def test_queues_returns_all_known_queues(client, monkeypatch):
def _fake():
return {
"queues": dict.fromkeys(activity_module._QUEUE_NAMES, 0),
"fetched_at": datetime.now(UTC).isoformat(),
}
monkeypatch.setattr(activity_module, "_read_queues_sync", _fake)
resp = await client.get("/api/system/activity/queues")
assert resp.status_code == 200
body = await resp.get_json()
assert set(body["queues"].keys()) == set(activity_module._QUEUE_NAMES)
@pytest.mark.asyncio
async def test_queues_cached_within_ttl(client, monkeypatch):
call_count = {"n": 0}
def _counting():
call_count["n"] += 1
return {"queues": {}, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_queues_sync", _counting)
await client.get("/api/system/activity/queues")
await client.get("/api/system/activity/queues")
await client.get("/api/system/activity/queues")
assert call_count["n"] == 1 # cached after the first call
@pytest.mark.asyncio
async def test_queues_redis_failure_returns_null_per_failing_queue(client, monkeypatch):
"""_read_queues_sync's per-queue try/except returns None on failure."""
def _partial():
out = dict.fromkeys(activity_module._QUEUE_NAMES, 0)
out["ml"] = None # simulate broker hiccup on the ml queue
return {"queues": out, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_queues_sync", _partial)
resp = await client.get("/api/system/activity/queues")
body = await resp.get_json()
assert body["queues"]["ml"] is None
assert body["queues"]["import"] == 0
# --- /workers ------------------------------------------------------
@pytest.mark.asyncio
async def test_workers_returns_per_worker_queue_membership(client, monkeypatch):
def _fake():
return {
"workers": {
"worker@host1": {"queues": ["import", "thumbnail"], "active_count": 2},
"worker@host2": {"queues": ["ml"], "active_count": 0},
},
"fetched_at": "x",
}
monkeypatch.setattr(activity_module, "_read_workers_sync", _fake)
resp = await client.get("/api/system/activity/workers")
assert resp.status_code == 200
body = await resp.get_json()
assert "worker@host1" in body["workers"]
assert "import" in body["workers"]["worker@host1"]["queues"]
@pytest.mark.asyncio
async def test_workers_cached_within_ttl(client, monkeypatch):
call_count = {"n": 0}
def _counting():
call_count["n"] += 1
return {"workers": {}, "fetched_at": "x"}
monkeypatch.setattr(activity_module, "_read_workers_sync", _counting)
await client.get("/api/system/activity/workers")
await client.get("/api/system/activity/workers")
assert call_count["n"] == 1
# --- /runs ---------------------------------------------------------
@pytest_asyncio.fixture
async def _seed_runs(db):
"""Insert 5 task_run rows for paging tests."""
now = datetime.now(UTC)
for i in range(5):
db.add(TaskRun(
celery_task_id=f"tid-{i}",
queue="import" if i % 2 == 0 else "ml",
task_name=f"backend.app.tasks.fake.task_{i}",
target_id=i,
started_at=now - timedelta(seconds=10 - i),
finished_at=now - timedelta(seconds=9 - i),
duration_ms=1000,
status="ok" if i < 3 else "error",
error_type="ValueError" if i >= 3 else None,
error_message="boom" if i >= 3 else None,
))
await db.commit()
@pytest.mark.asyncio
async def test_runs_paginated_descending_by_id(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?limit=3")
body = await resp.get_json()
assert len(body["runs"]) == 3
# Descending: latest id first.
ids = [r["id"] for r in body["runs"]]
assert ids == sorted(ids, reverse=True)
assert body["next_cursor"] is not None # 5 total, asked for 3 → more
@pytest.mark.asyncio
async def test_runs_filter_by_queue(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?queue=ml")
body = await resp.get_json()
assert all(r["queue"] == "ml" for r in body["runs"])
assert len(body["runs"]) == 2 # i=1, i=3
@pytest.mark.asyncio
async def test_runs_filter_by_status(client, _seed_runs):
resp = await client.get("/api/system/activity/runs?status=error")
body = await resp.get_json()
assert all(r["status"] == "error" for r in body["runs"])
assert len(body["runs"]) == 2 # i=3, i=4
@pytest.mark.asyncio
async def test_runs_keyset_cursor(client, _seed_runs):
page1 = await (await client.get("/api/system/activity/runs?limit=2")).get_json()
assert len(page1["runs"]) == 2
assert page1["next_cursor"] is not None
page2 = await (await client.get(
f"/api/system/activity/runs?limit=2&before_id={page1['next_cursor']}"
)).get_json()
assert len(page2["runs"]) == 2
page1_ids = {r["id"] for r in page1["runs"]}
page2_ids = {r["id"] for r in page2["runs"]}
assert page1_ids.isdisjoint(page2_ids)
@pytest.mark.asyncio
async def test_runs_invalid_limit_400(client):
resp = await client.get("/api/system/activity/runs?limit=not-a-number")
assert resp.status_code == 400
# --- /failures -----------------------------------------------------
@pytest_asyncio.fixture
async def _seed_failures(db):
"""Mix of ok / error / timeout rows; some old, some recent."""
now = datetime.now(UTC)
recent = [
("error", "OperationalError"),
("error", "OperationalError"),
("error", "OperationalError"),
("timeout", "TimeoutError"),
("timeout", "TimeoutError"),
("error", "OSError"),
("ok", None), # success — should NOT appear in failures
]
for i, (status, etype) in enumerate(recent):
db.add(TaskRun(
celery_task_id=f"f-{i}",
queue="ml",
task_name="backend.app.tasks.fake.x",
target_id=i,
started_at=now - timedelta(seconds=120 - i),
finished_at=now - timedelta(seconds=60 - i),
duration_ms=1000,
status=status,
error_type=etype,
error_message="boom" if status != "ok" else None,
))
# One ancient failure (>24h) that should NOT appear.
db.add(TaskRun(
celery_task_id="f-old",
queue="ml",
task_name="backend.app.tasks.fake.x",
target_id=999,
started_at=now - timedelta(hours=30),
finished_at=now - timedelta(hours=29),
duration_ms=1000,
status="error",
error_type="OldError",
error_message="ancient",
))
await db.commit()
@pytest.mark.asyncio
async def test_failures_returns_recent_errors_and_timeouts(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
statuses = {r["status"] for r in body["recent"]}
assert statuses == {"error", "timeout"}
# No ok rows.
assert "ok" not in statuses
@pytest.mark.asyncio
async def test_failures_count_by_type_groups_correctly(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
counts = body["count_by_type"]
assert counts.get("OperationalError") == 3
assert counts.get("TimeoutError") == 2
assert counts.get("OSError") == 1
assert "OldError" not in counts # outside 24h window
@pytest.mark.asyncio
async def test_failures_only_within_24h_window(client, _seed_failures):
resp = await client.get("/api/system/activity/failures")
body = await resp.get_json()
ids = {r["error_type"] for r in body["recent"]}
assert "OldError" not in ids
+171
View File
@@ -93,3 +93,174 @@ def test_cleanup_old_deletes_finished_old(db_sync):
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
assert remaining == {"/import/b.jpg", "/import/c.jpg"}
# --- FC-3i: task_run sweep + retention -----------------------------
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
error_type=None):
from backend.app.models import TaskRun
row = TaskRun(
celery_task_id="x",
queue="ml",
task_name="backend.app.tasks.fake.t",
target_id=1,
started_at=started_at,
finished_at=finished_at,
duration_ms=1000 if finished_at else None,
status=status,
error_type=error_type,
error_message="x" if status in ("error", "timeout") else None,
)
db_sync.add(row)
db_sync.flush()
return row.id
def test_recover_stalled_task_runs_flips_old_running_to_error(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import recover_stalled_task_runs
now = datetime.now(UTC)
stale_id = _make_task_run(
db_sync, status="running", started_at=now - timedelta(minutes=10),
)
db_sync.commit()
recovered = recover_stalled_task_runs.apply().get()
assert recovered == 1
db_sync.expire_all()
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.id == stale_id)
).scalar_one()
error_type = db_sync.execute(
select(TaskRun.error_type).where(TaskRun.id == stale_id)
).scalar_one()
assert status == "error"
assert error_type == "RecoverySweep"
def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import recover_stalled_task_runs
now = datetime.now(UTC)
fresh_id = _make_task_run(
db_sync, status="running", started_at=now - timedelta(seconds=30),
)
db_sync.commit()
recovered = recover_stalled_task_runs.apply().get()
assert recovered == 0
db_sync.expire_all()
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.id == fresh_id)
).scalar_one()
assert status == "running"
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
old_id = _make_task_run(
db_sync, status="ok",
started_at=now - timedelta(hours=30),
finished_at=now - timedelta(hours=29),
)
recent_id = _make_task_run(
db_sync, status="ok",
started_at=now - timedelta(hours=2),
finished_at=now - timedelta(hours=1),
)
db_sync.commit()
result = prune_task_runs.apply().get()
assert result["ok_deleted"] == 1
db_sync.expire_all()
surviving_ids = set(db_sync.execute(
select(TaskRun.id).where(TaskRun.id.in_([old_id, recent_id]))
).scalars().all())
assert surviving_ids == {recent_id}
def test_prune_task_runs_deletes_failures_older_than_7d(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
old_id = _make_task_run(
db_sync, status="error", error_type="OldError",
started_at=now - timedelta(days=10),
finished_at=now - timedelta(days=9),
)
db_sync.commit()
result = prune_task_runs.apply().get()
assert result["failures_deleted"] >= 1
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == old_id)
).scalar_one_or_none()
assert surviving is None
def test_prune_task_runs_keeps_recent_failures(db_sync):
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
recent_id = _make_task_run(
db_sync, status="error", error_type="RecentError",
started_at=now - timedelta(days=6),
finished_at=now - timedelta(days=5),
)
db_sync.commit()
prune_task_runs.apply().get()
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == recent_id)
).scalar_one_or_none()
assert surviving == recent_id
def test_prune_task_runs_never_deletes_running(db_sync):
"""Even a 30-day-old running row stays — recovery sweep is the
mechanism that flips them; prune doesn't touch in-flight."""
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import prune_task_runs
now = datetime.now(UTC)
ancient_id = _make_task_run(
db_sync, status="running",
started_at=now - timedelta(days=30),
)
db_sync.commit()
prune_task_runs.apply().get()
db_sync.expire_all()
surviving = db_sync.execute(
select(TaskRun.id).where(TaskRun.id == ancient_id)
).scalar_one_or_none()
assert surviving == ancient_id
+258
View File
@@ -0,0 +1,258 @@
"""FC-3i: Celery signal → task_run row tests.
Uses task_always_eager so signals fire synchronously in-process during
the test. Scoped autouse fixture keeps the eager mode confined to this
file (other tests may break with eager mode enabled globally).
Test assertions on signal-mutated rows use COLUMN SELECTS / db_sync.
scalar (per reference_async_coredml_test_assertions). ORM attribute
access on a row a signal handler just mutated risks MissingGreenlet
across async/sync boundaries.
"""
import pytest
from sqlalchemy import func, select
from backend.app.celery_app import celery
from backend.app.models import TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture(autouse=True)
def _eager_celery(monkeypatch):
"""task_always_eager so signals fire in-process during the test."""
monkeypatch.setattr(celery.conf, "task_always_eager", True)
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
@pytest.mark.asyncio
async def test_prerun_inserts_running_row(db_sync):
"""Fire a no-op task; assert task_run row inserted with running status."""
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_999) # nonexistent image_id → no-op
rows = db_sync.execute(
select(TaskRun.queue, TaskRun.target_id, TaskRun.task_name)
.where(TaskRun.target_id == 999_999)
).all()
assert len(rows) == 1
queue, target_id, task_name = rows[0]
assert queue == "thumbnail"
assert target_id == 999_999
assert task_name.endswith(".generate_thumbnail")
@pytest.mark.asyncio
async def test_postrun_marks_ok_with_duration(db_sync):
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_998)
row = db_sync.execute(
select(
TaskRun.status, TaskRun.finished_at, TaskRun.duration_ms,
).where(TaskRun.target_id == 999_998)
).one()
status, finished_at, duration_ms = row
assert status == "ok"
assert finished_at is not None
assert duration_ms is not None and duration_ms >= 0
@pytest.mark.asyncio
async def test_failure_records_error_type_and_message(db_sync, monkeypatch):
"""Force a tracked task to raise; assert error fields populated."""
from backend.app.tasks.thumbnail import generate_thumbnail
def _explode(*a, **k):
raise ValueError("synthetic failure")
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run",
_explode,
)
with pytest.raises(ValueError):
generate_thumbnail.delay(999_997).get(propagate=True)
row = db_sync.execute(
select(
TaskRun.status, TaskRun.error_type, TaskRun.error_message,
TaskRun.finished_at,
).where(TaskRun.target_id == 999_997)
.order_by(TaskRun.id.desc()).limit(1)
).one()
status, error_type, error_message, finished_at = row
assert status == "error"
assert error_type == "ValueError"
assert "synthetic failure" in (error_message or "")
assert finished_at is not None
@pytest.mark.asyncio
async def test_soft_time_limit_records_timeout_status(db_sync, monkeypatch):
from celery.exceptions import SoftTimeLimitExceeded
from backend.app.tasks.thumbnail import generate_thumbnail
def _timeout(*a, **k):
raise SoftTimeLimitExceeded("simulated soft limit")
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run",
_timeout,
)
with pytest.raises(SoftTimeLimitExceeded):
generate_thumbnail.delay(999_996).get(propagate=True)
status = db_sync.execute(
select(TaskRun.status).where(TaskRun.target_id == 999_996)
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert status == "timeout"
@pytest.mark.asyncio
async def test_untracked_celery_internal_tasks_skip(db_sync):
"""celery.chord_unlock is in the untracked list; firing it should
NOT insert a task_run row."""
before = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
# Send a chord_unlock-style task via apply (no real task to dispatch).
# The signal handler checks task.name against _UNTRACKED_TASK_NAMES;
# if name is in the list, no row is inserted. We verify by firing
# a regular task and asserting only ONE row appears, not two.
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_995)
after = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
assert after - before == 1
@pytest.mark.asyncio
async def test_args_summary_truncated_to_255(db_sync, monkeypatch):
"""Task with a very large repr(args); assert args_summary <= 255."""
from backend.app.tasks.thumbnail import generate_thumbnail
# Force a giant arg by patching repr(args) target — actually just
# call with a large int that produces a long repr.
big_id = int("9" * 200) # 200-char repr
# The signal handler reads repr(args), so this naturally produces
# a long args_summary that should get truncated to 255.
try:
generate_thumbnail.delay(big_id).get(timeout=5, propagate=False)
except Exception:
pass
args_summary = db_sync.execute(
select(TaskRun.args_summary)
.where(TaskRun.task_name.endswith(".generate_thumbnail"))
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert args_summary is None or len(args_summary) <= 255
@pytest.mark.asyncio
async def test_error_message_truncated_to_2000(db_sync, monkeypatch):
from backend.app.tasks.thumbnail import generate_thumbnail
huge_message = "X" * 5000
def _raise_huge(*a, **k):
raise RuntimeError(huge_message)
monkeypatch.setattr(
"backend.app.tasks.thumbnail.generate_thumbnail.run", _raise_huge,
)
with pytest.raises(RuntimeError):
generate_thumbnail.delay(999_994).get(propagate=True)
error_message = db_sync.execute(
select(TaskRun.error_message).where(TaskRun.target_id == 999_994)
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert error_message is not None
assert len(error_message) <= 2000
@pytest.mark.asyncio
async def test_signal_handler_failure_does_not_break_task(db_sync, monkeypatch):
"""Monkeypatch sync_session_factory in the signals module to raise;
assert the task still returns its result and the worker doesn't
propagate the signal-handler exception."""
from backend.app.tasks.thumbnail import generate_thumbnail
def _broken_factory():
raise RuntimeError("db down")
monkeypatch.setattr(
"backend.app.celery_signals.sync_session_factory", _broken_factory,
)
# Task should still execute (no-op on missing image) without raising.
result = generate_thumbnail.delay(999_993).get(timeout=5)
assert result is not None # task ran; signal handler errors swallowed
@pytest.mark.parametrize("args,expected", [
((42,), 42),
((42, "ignored"), 42),
(("123",), 123),
(("not-int",), None),
((), None),
(None, None),
])
def test_target_id_extracted_from_first_int_arg(args, expected):
"""Unit test the helper directly; no DB needed."""
from backend.app.celery_signals import _target_id_from_args
assert _target_id_from_args(args) == expected
def test_finalize_sets_retry_status_with_retry_count(db_sync):
"""Direct unit-shape test of _finalize's retry path. We can't
reliably trigger Celery's task_retry signal under eager mode with
monkeypatched .run (the bind=True 'self' injection breaks), so
test the finalize helper directly — that's the code the retry
signal handler delegates to anyway."""
from datetime import UTC, datetime
from backend.app.celery_signals import _finalize
from backend.app.models import TaskRun
# Seed a 'running' row that _finalize can target.
row = TaskRun(
celery_task_id="retry-test-tid",
queue="ml",
task_name="backend.app.tasks.fake.t",
target_id=999_992,
started_at=datetime.now(UTC),
status="running",
)
db_sync.add(row)
db_sync.commit()
_finalize(
"retry-test-tid", status="retry",
error_type="RuntimeError",
error_message="first attempt",
retry_count=1,
)
# Column-select assertions — _finalize mutated the row via its own
# session; re-read fresh state through db_sync.
db_sync.expire_all()
status, retry_count, error_type = db_sync.execute(
select(TaskRun.status, TaskRun.retry_count, TaskRun.error_type)
.where(TaskRun.celery_task_id == "retry-test-tid")
).one()
assert status == "retry"
assert retry_count == 1
assert error_type == "RuntimeError"
@pytest.mark.asyncio
async def test_worker_hostname_recorded(db_sync):
"""Sender's hostname should land in worker_hostname."""
from backend.app.tasks.thumbnail import generate_thumbnail
generate_thumbnail.delay(999_991)
hostname = db_sync.execute(
select(TaskRun.worker_hostname).where(TaskRun.target_id == 999_991)
).scalar_one()
# Under eager mode hostname may be None or "celery@local"; just
# assert the column is touched (either string or None — never throws).
assert hostname is None or isinstance(hostname, str)