Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d1c0b82a22 | |||
| 37fcc74954 | |||
| 6a532c1497 | |||
| 368613068a | |||
| ddbb84d8aa | |||
| 9b252948f9 | |||
| 89d0cb2124 | |||
| 36611cbe00 | |||
| 48ef22445a | |||
| e523d0ac94 | |||
| 541e2bfe6a | |||
| 7782672a51 | |||
| d12b51f6b7 | |||
| 0cda46fcdb | |||
| 79fee98db4 | |||
| b1d68929c5 | |||
| 553567738e |
@@ -0,0 +1,86 @@
|
|||||||
|
"""fc3i: task_run table
|
||||||
|
|
||||||
|
Revision ID: 0016
|
||||||
|
Revises: 0015
|
||||||
|
Create Date: 2026-05-24
|
||||||
|
|
||||||
|
Additive only. New table records every Celery task attempt via signal
|
||||||
|
handlers (backend.app.celery_signals). Status is plain String(16) not
|
||||||
|
Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard-
|
||||||
|
fail at INSERT, String columns extend cleanly).
|
||||||
|
|
||||||
|
Composite indexes anticipate the three dashboard panes:
|
||||||
|
- (queue, started_at desc) — per-lane recent activity
|
||||||
|
- (status, started_at desc) — recent failures pane
|
||||||
|
- (task_name, started_at desc) — drill-down by task
|
||||||
|
|
||||||
|
Indexed columns get individual indexes via `index=True` on the model;
|
||||||
|
the composites below cover the multi-column lookups.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0016"
|
||||||
|
down_revision: Union[str, None] = "0015"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"task_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("celery_task_id", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("queue", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("task_name", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("target_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), nullable=False,
|
||||||
|
server_default="running",
|
||||||
|
),
|
||||||
|
sa.Column("error_type", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text(), nullable=True),
|
||||||
|
sa.Column("retry_count", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("worker_hostname", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("args_summary", sa.String(length=255), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Single-column indexes (matches Mapped[...].index=True on model).
|
||||||
|
op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"])
|
||||||
|
op.create_index("ix_task_run_queue", "task_run", ["queue"])
|
||||||
|
op.create_index("ix_task_run_task_name", "task_run", ["task_name"])
|
||||||
|
op.create_index("ix_task_run_started_at", "task_run", ["started_at"])
|
||||||
|
op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"])
|
||||||
|
op.create_index("ix_task_run_status", "task_run", ["status"])
|
||||||
|
|
||||||
|
# Composite indexes for dashboard query patterns.
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_queue_started",
|
||||||
|
"task_run", ["queue", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_status_started",
|
||||||
|
"task_run", ["status", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_task_run_name_started",
|
||||||
|
"task_run", ["task_name", sa.text("started_at DESC")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_task_run_name_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_status_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_queue_started", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_status", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_finished_at", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_started_at", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_task_name", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_queue", table_name="task_run")
|
||||||
|
op.drop_index("ix_task_run_celery_task_id", table_name="task_run")
|
||||||
|
op.drop_table("task_run")
|
||||||
@@ -33,6 +33,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .showcase import showcase_bp
|
from .showcase import showcase_bp
|
||||||
from .sources import sources_bp
|
from .sources import sources_bp
|
||||||
from .suggestions import suggestions_bp
|
from .suggestions import suggestions_bp
|
||||||
|
from .system_activity import system_activity_bp
|
||||||
from .tags import tags_bp
|
from .tags import tags_bp
|
||||||
return [
|
return [
|
||||||
api_bp,
|
api_bp,
|
||||||
@@ -44,6 +45,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
artists_bp,
|
artists_bp,
|
||||||
showcase_bp,
|
showcase_bp,
|
||||||
settings_bp,
|
settings_bp,
|
||||||
|
system_activity_bp,
|
||||||
import_admin_bp,
|
import_admin_bp,
|
||||||
migrate_bp,
|
migrate_bp,
|
||||||
suggestions_bp,
|
suggestions_bp,
|
||||||
|
|||||||
@@ -24,7 +24,12 @@ _VALID_KINDS = frozenset({
|
|||||||
"ml_queue", "verify", "rollback", "cleanup",
|
"ml_queue", "verify", "rollback", "cleanup",
|
||||||
})
|
})
|
||||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
|
# Backup gate retired 2026-05-24 — operator-flagged the speculative-safety
|
||||||
|
# requirement was actively blocking the UI ingest path (backup itself is
|
||||||
|
# unreliable on large NFS-backed image libraries) and FC-3h will rewrite
|
||||||
|
# the backup surface as a first-class feature with its own scheduling
|
||||||
|
# + recovery. Leaving the constant for historical grep.
|
||||||
|
_APPLY_KINDS: frozenset[str] = frozenset()
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""FC-3i: system activity dashboard endpoints.
|
||||||
|
|
||||||
|
Read-only. Combines Redis-broker queue depths (LLEN per queue),
|
||||||
|
Celery worker introspection (celery inspect), and the task_run DB
|
||||||
|
history into the surfaces the SystemActivityTab UI consumes.
|
||||||
|
|
||||||
|
All filesystem/sync-client work goes through asyncio.to_thread per
|
||||||
|
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import desc, func, select
|
||||||
|
|
||||||
|
from ..config import get_config
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import TaskRun
|
||||||
|
|
||||||
|
system_activity_bp = Blueprint(
|
||||||
|
"system_activity", __name__, url_prefix="/api/system/activity",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Canonical queue order — must match celery_app.task_routes. UI renders
|
||||||
|
# in this order; queues with no LLEN response show as null rather than
|
||||||
|
# absent.
|
||||||
|
_QUEUE_NAMES = (
|
||||||
|
"default", "import", "thumbnail", "ml",
|
||||||
|
"download", "scan", "maintenance",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Cache module-level so all requests share the cache between polls.
|
||||||
|
# Tests can reset via direct dict mutation if needed.
|
||||||
|
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
|
||||||
|
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
|
||||||
|
_QUEUE_CACHE_TTL = 2.0
|
||||||
|
_WORKER_CACHE_TTL = 5.0
|
||||||
|
|
||||||
|
|
||||||
|
def _read_queues_sync() -> dict:
|
||||||
|
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
|
||||||
|
asyncio.to_thread. Per-queue try/except returns None on failure so
|
||||||
|
one bad queue doesn't break the whole response."""
|
||||||
|
import redis # local import; only this endpoint needs it
|
||||||
|
|
||||||
|
cfg = get_config()
|
||||||
|
client = redis.Redis.from_url(cfg.celery_broker_url)
|
||||||
|
out: dict = {}
|
||||||
|
for name in _QUEUE_NAMES:
|
||||||
|
try:
|
||||||
|
out[name] = int(client.llen(name))
|
||||||
|
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
|
||||||
|
out[name] = None
|
||||||
|
return {
|
||||||
|
"queues": out,
|
||||||
|
"fetched_at": datetime.now(UTC).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _read_workers_sync() -> dict:
|
||||||
|
"""celery inspect active_queues + active. Returns per-worker info."""
|
||||||
|
from ..celery_app import celery as celery_app
|
||||||
|
|
||||||
|
insp = celery_app.control.inspect(timeout=2.0)
|
||||||
|
active_queues = insp.active_queues() or {}
|
||||||
|
active_tasks = insp.active() or {}
|
||||||
|
|
||||||
|
workers: dict = {}
|
||||||
|
for hostname, queues in active_queues.items():
|
||||||
|
workers[hostname] = {
|
||||||
|
"queues": sorted({q["name"] for q in queues}),
|
||||||
|
"active_count": len(active_tasks.get(hostname, [])),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"workers": workers,
|
||||||
|
"fetched_at": datetime.now(UTC).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/queues", methods=["GET"])
|
||||||
|
async def get_queues():
|
||||||
|
"""Per-queue Redis LLEN. Cached 2s.
|
||||||
|
|
||||||
|
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
||||||
|
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
||||||
|
_QUEUE_CACHE["ts"] = now
|
||||||
|
return jsonify(_QUEUE_CACHE["data"])
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/workers", methods=["GET"])
|
||||||
|
async def get_workers():
|
||||||
|
"""Live celery inspect. Cached 5s.
|
||||||
|
|
||||||
|
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
|
||||||
|
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
|
||||||
|
_WORKER_CACHE["ts"] = now
|
||||||
|
return jsonify(_WORKER_CACHE["data"])
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/runs", methods=["GET"])
|
||||||
|
async def list_runs():
|
||||||
|
"""Paginated task_run history. Query params:
|
||||||
|
queue=<name> filter to one queue
|
||||||
|
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||||
|
limit=<int> default 50, max 200
|
||||||
|
before_id=<int> cursor for keyset pagination
|
||||||
|
|
||||||
|
Response: {runs: [...], next_cursor: id|null}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "50")), 200)
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
if limit < 1:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
|
||||||
|
queue = request.args.get("queue")
|
||||||
|
status = request.args.get("status")
|
||||||
|
before_id_raw = request.args.get("before_id")
|
||||||
|
before_id = int(before_id_raw) if before_id_raw else None
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
stmt = select(TaskRun).order_by(desc(TaskRun.id))
|
||||||
|
if queue:
|
||||||
|
stmt = stmt.where(TaskRun.queue == queue)
|
||||||
|
if status:
|
||||||
|
stmt = stmt.where(TaskRun.status == status)
|
||||||
|
if before_id is not None:
|
||||||
|
stmt = stmt.where(TaskRun.id < before_id)
|
||||||
|
stmt = stmt.limit(limit + 1)
|
||||||
|
rows = (await session.execute(stmt)).scalars().all()
|
||||||
|
|
||||||
|
has_more = len(rows) > limit
|
||||||
|
rows = rows[:limit]
|
||||||
|
return jsonify({
|
||||||
|
"runs": [_row_to_dict(r) for r in rows],
|
||||||
|
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/failures", methods=["GET"])
|
||||||
|
async def list_failures():
|
||||||
|
"""Recent failures across all lanes (24h window).
|
||||||
|
|
||||||
|
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "50")), 200)
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "invalid_limit"}), 400
|
||||||
|
|
||||||
|
since = datetime.now(UTC) - timedelta(hours=24)
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
recent_stmt = (
|
||||||
|
select(TaskRun)
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||||
|
.where(TaskRun.finished_at >= since)
|
||||||
|
.order_by(desc(TaskRun.finished_at))
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
recent = (await session.execute(recent_stmt)).scalars().all()
|
||||||
|
|
||||||
|
count_stmt = (
|
||||||
|
select(TaskRun.error_type, func.count(TaskRun.id))
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||||
|
.where(TaskRun.finished_at >= since)
|
||||||
|
.group_by(TaskRun.error_type)
|
||||||
|
.order_by(desc(func.count(TaskRun.id)))
|
||||||
|
)
|
||||||
|
counts = (await session.execute(count_stmt)).all()
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
"recent": [_row_to_dict(r) for r in recent],
|
||||||
|
"count_by_type": {
|
||||||
|
(row[0] or "Unknown"): row[1]
|
||||||
|
for row in counts
|
||||||
|
},
|
||||||
|
"since": since.isoformat(),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _row_to_dict(r: TaskRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": r.id,
|
||||||
|
"queue": r.queue,
|
||||||
|
"task_name": r.task_name,
|
||||||
|
"target_id": r.target_id,
|
||||||
|
"celery_task_id": r.celery_task_id,
|
||||||
|
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||||
|
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||||
|
"duration_ms": r.duration_ms,
|
||||||
|
"status": r.status,
|
||||||
|
"error_type": r.error_type,
|
||||||
|
"error_message": r.error_message,
|
||||||
|
"retry_count": r.retry_count,
|
||||||
|
"worker_hostname": r.worker_hostname,
|
||||||
|
"args_summary": r.args_summary,
|
||||||
|
}
|
||||||
@@ -81,9 +81,20 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
|
"recover-stalled-task-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
||||||
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
||||||
|
},
|
||||||
|
"prune-task-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.prune_task_runs",
|
||||||
|
"schedule": 86400.0, # daily
|
||||||
|
},
|
||||||
},
|
},
|
||||||
timezone="UTC",
|
timezone="UTC",
|
||||||
)
|
)
|
||||||
|
# FC-3i: register task_run signal handlers (side-effect import).
|
||||||
|
from . import celery_signals # noqa: F401
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""FC-3i: task_run lifecycle via Celery signals.
|
||||||
|
|
||||||
|
Subscribes to task_prerun / task_postrun / task_failure / task_retry
|
||||||
|
and persists one task_run row per task attempt. Drop-in for every
|
||||||
|
existing and future Celery task — no per-task instrumentation.
|
||||||
|
|
||||||
|
Signal handlers run inside the worker process (sync context); DB
|
||||||
|
writes go through the existing shared sync engine
|
||||||
|
(backend.app.tasks._sync_engine.sync_session_factory) — one engine
|
||||||
|
per worker process, not per-task, so we don't blow Postgres
|
||||||
|
max_connections under load (the reason FC-3g shared-engine fix
|
||||||
|
existed).
|
||||||
|
|
||||||
|
Failure-mode discipline (operator-pressed point): every handler is
|
||||||
|
wrapped in try/except that swallows + logs. If the DB is down or the
|
||||||
|
handler has a bug, the real task still runs — the dashboard goes
|
||||||
|
dark for that interval. Monitoring NEVER breaks the thing it's
|
||||||
|
monitoring.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
||||||
|
|
||||||
|
from .models import TaskRun
|
||||||
|
from .tasks._sync_engine import sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Celery-internal tasks that would generate dashboard noise without
|
||||||
|
# operational value. Conservative list; extend only when a specific
|
||||||
|
# task proves noisy.
|
||||||
|
_UNTRACKED_TASK_NAMES = frozenset({
|
||||||
|
"celery.chord_unlock",
|
||||||
|
"celery.backend_cleanup",
|
||||||
|
"celery.chunks",
|
||||||
|
})
|
||||||
|
|
||||||
|
_MAX_ERROR_MESSAGE_LEN = 2000
|
||||||
|
_MAX_ARGS_SUMMARY_LEN = 255
|
||||||
|
_MAX_WORKER_HOSTNAME_LEN = 128
|
||||||
|
|
||||||
|
# PostgreSQL Integer is signed 32-bit. Tasks called with a first-arg
|
||||||
|
# int outside this range (e.g. an absurdly large mock value, or a
|
||||||
|
# string-of-digits coercible to int but bigger than 2^31-1) would crash
|
||||||
|
# the INSERT with NumericValueOutOfRange. Bound the recorded value to
|
||||||
|
# the column's range; values outside become None (target_id is
|
||||||
|
# nullable, so this is safe).
|
||||||
|
_INT32_MAX = 2_147_483_647
|
||||||
|
_INT32_MIN = -2_147_483_648
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_for(task) -> str:
|
||||||
|
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||||
|
Keep in sync if task_routes is reordered."""
|
||||||
|
name = getattr(task, "name", "") or ""
|
||||||
|
if name.startswith("backend.app.tasks.import_file."):
|
||||||
|
return "import"
|
||||||
|
if name.startswith("backend.app.tasks.ml."):
|
||||||
|
return "ml"
|
||||||
|
if name.startswith("backend.app.tasks.thumbnail."):
|
||||||
|
return "thumbnail"
|
||||||
|
if name.startswith("backend.app.tasks.download."):
|
||||||
|
return "download"
|
||||||
|
if name.startswith("backend.app.tasks.scan."):
|
||||||
|
return "scan"
|
||||||
|
if name.startswith((
|
||||||
|
"backend.app.tasks.maintenance.",
|
||||||
|
"backend.app.tasks.migration.",
|
||||||
|
)):
|
||||||
|
return "maintenance"
|
||||||
|
return "default"
|
||||||
|
|
||||||
|
|
||||||
|
def _target_id_from_args(args) -> int | None:
|
||||||
|
"""Best-effort: if the first positional arg parses as int AND fits
|
||||||
|
in the column's signed-32-bit range, record it as target_id
|
||||||
|
(image_id, source_id, etc.). Never raises."""
|
||||||
|
if not args:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = int(args[0])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if value < _INT32_MIN or value > _INT32_MAX:
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _truncate(s, limit: int) -> str | None:
|
||||||
|
if s is None:
|
||||||
|
return None
|
||||||
|
text = str(s)
|
||||||
|
return text if len(text) <= limit else text[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_tracked(task_name: str | None) -> bool:
|
||||||
|
return bool(task_name) and task_name not in _UNTRACKED_TASK_NAMES
|
||||||
|
|
||||||
|
|
||||||
|
@task_prerun.connect
|
||||||
|
def _on_prerun(sender=None, task_id=None, task=None, args=None,
|
||||||
|
kwargs=None, **_):
|
||||||
|
if not _is_tracked(getattr(task, "name", None)):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
Session = sync_session_factory()
|
||||||
|
with Session() as session:
|
||||||
|
session.add(TaskRun(
|
||||||
|
celery_task_id=task_id or "",
|
||||||
|
queue=_queue_for(task),
|
||||||
|
task_name=task.name,
|
||||||
|
target_id=_target_id_from_args(args),
|
||||||
|
started_at=datetime.now(UTC),
|
||||||
|
status="running",
|
||||||
|
args_summary=_truncate(repr(args), _MAX_ARGS_SUMMARY_LEN),
|
||||||
|
worker_hostname=_truncate(
|
||||||
|
getattr(sender, "hostname", None),
|
||||||
|
_MAX_WORKER_HOSTNAME_LEN,
|
||||||
|
),
|
||||||
|
))
|
||||||
|
session.commit()
|
||||||
|
except Exception: # noqa: BLE001 — never break the worker
|
||||||
|
log.exception("task_run prerun insert failed (task=%s)",
|
||||||
|
getattr(task, "name", "?"))
|
||||||
|
|
||||||
|
|
||||||
|
def _finalize(task_id: str, *, status: str,
|
||||||
|
error_type: str | None = None,
|
||||||
|
error_message: str | None = None,
|
||||||
|
retry_count: int | None = None) -> None:
|
||||||
|
"""Shared write path for postrun/failure/retry. Picks the most-
|
||||||
|
recent task_run row for this celery_task_id that's still 'running'
|
||||||
|
(retries reuse the same celery_task_id; each new attempt's prerun
|
||||||
|
inserts a fresh row, so finalize targets the latest running row)."""
|
||||||
|
try:
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
Session = sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with Session() as session:
|
||||||
|
row = session.execute(
|
||||||
|
select(TaskRun)
|
||||||
|
.where(TaskRun.celery_task_id == task_id)
|
||||||
|
.where(TaskRun.status == "running")
|
||||||
|
.order_by(TaskRun.id.desc())
|
||||||
|
.limit(1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if row is None:
|
||||||
|
return # no prerun row (untracked, insert failed, or already finalized)
|
||||||
|
row.finished_at = now
|
||||||
|
row.duration_ms = int(
|
||||||
|
(now - row.started_at).total_seconds() * 1000
|
||||||
|
)
|
||||||
|
row.status = status
|
||||||
|
if error_type is not None:
|
||||||
|
row.error_type = _truncate(error_type, 128)
|
||||||
|
if error_message is not None:
|
||||||
|
row.error_message = _truncate(error_message, _MAX_ERROR_MESSAGE_LEN)
|
||||||
|
if retry_count is not None:
|
||||||
|
row.retry_count = retry_count
|
||||||
|
session.commit()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.exception("task_run finalize failed (task_id=%s)", task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@task_postrun.connect
|
||||||
|
def _on_postrun(sender=None, task_id=None, task=None, args=None,
|
||||||
|
kwargs=None, retval=None, state=None, **_):
|
||||||
|
if not _is_tracked(getattr(task, "name", None)):
|
||||||
|
return
|
||||||
|
# state is one of SUCCESS/FAILURE/RETRY/etc. Only handle SUCCESS;
|
||||||
|
# task_failure handles FAILURE explicitly (with the exception).
|
||||||
|
if state != "SUCCESS":
|
||||||
|
return
|
||||||
|
_finalize(task_id, status="ok")
|
||||||
|
|
||||||
|
|
||||||
|
@task_failure.connect
|
||||||
|
def _on_failure(sender=None, task_id=None, exception=None,
|
||||||
|
args=None, kwargs=None, einfo=None, **_):
|
||||||
|
if not _is_tracked(getattr(sender, "name", None)):
|
||||||
|
return
|
||||||
|
status = ("timeout"
|
||||||
|
if isinstance(exception, SoftTimeLimitExceeded)
|
||||||
|
else "error")
|
||||||
|
_finalize(
|
||||||
|
task_id, status=status,
|
||||||
|
error_type=type(exception).__name__ if exception else "Unknown",
|
||||||
|
error_message=str(exception) if exception else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@task_retry.connect
|
||||||
|
def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
||||||
|
if not _is_tracked(getattr(sender, "name", None)):
|
||||||
|
return
|
||||||
|
task_id = getattr(request, "id", None)
|
||||||
|
if not task_id:
|
||||||
|
return
|
||||||
|
# Mark current attempt's row as 'retry' (terminal for this row).
|
||||||
|
# The next attempt's task_prerun inserts a fresh row.
|
||||||
|
_finalize(
|
||||||
|
task_id, status="retry",
|
||||||
|
error_type=type(reason).__name__ if reason else "Retry",
|
||||||
|
error_message=str(reason) if reason else None,
|
||||||
|
retry_count=getattr(request, "retries", 0),
|
||||||
|
)
|
||||||
@@ -21,6 +21,7 @@ from .tag_alias import TagAlias
|
|||||||
from .tag_allowlist import TagAllowlist
|
from .tag_allowlist import TagAllowlist
|
||||||
from .tag_reference_embedding import TagReferenceEmbedding
|
from .tag_reference_embedding import TagReferenceEmbedding
|
||||||
from .tag_suggestion_rejection import TagSuggestionRejection
|
from .tag_suggestion_rejection import TagSuggestionRejection
|
||||||
|
from .task_run import TaskRun
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
@@ -46,4 +47,5 @@ __all__ = [
|
|||||||
"TagAllowlist",
|
"TagAllowlist",
|
||||||
"TagReferenceEmbedding",
|
"TagReferenceEmbedding",
|
||||||
"TagSuggestionRejection",
|
"TagSuggestionRejection",
|
||||||
|
"TaskRun",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""FC-3i: task_run — per-Celery-task lifecycle audit row.
|
||||||
|
|
||||||
|
One row inserted by the task_prerun signal at task start, updated by
|
||||||
|
task_postrun / task_failure / task_retry. The shape supports the
|
||||||
|
SystemActivity dashboard's three panes: per-queue queue+worker summary,
|
||||||
|
recent failures (24h, grouped by error_type), and full paginated
|
||||||
|
activity history.
|
||||||
|
|
||||||
|
Retention: ok rows pruned after 24h, error/timeout after 7d (see
|
||||||
|
backend.app.tasks.maintenance.prune_task_runs).
|
||||||
|
|
||||||
|
Recovery: rows stuck in 'running' for >5 min flipped to 'error' by
|
||||||
|
backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Integer, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class TaskRun(Base):
|
||||||
|
__tablename__ = "task_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
celery_task_id: Mapped[str] = mapped_column(
|
||||||
|
String(64), nullable=False, index=True,
|
||||||
|
)
|
||||||
|
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||||
|
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||||
|
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True,
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True,
|
||||||
|
)
|
||||||
|
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
)
|
||||||
|
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
retry_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
worker_hostname: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
args_summary: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
@@ -9,7 +9,7 @@ from PIL import Image
|
|||||||
from sqlalchemy import delete, select, update
|
from sqlalchemy import delete, select, update
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask
|
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
||||||
from ..utils.phash import compute_phash
|
from ..utils.phash import compute_phash
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
@@ -20,6 +20,8 @@ OLD_TASK_DAYS = 7
|
|||||||
PHASH_PAGE = 500
|
PHASH_PAGE = 500
|
||||||
VERIFY_PAGE = 200
|
VERIFY_PAGE = 200
|
||||||
FFPROBE_TIMEOUT_SECONDS = 10
|
FFPROBE_TIMEOUT_SECONDS = 10
|
||||||
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||||
@@ -80,6 +82,71 @@ def cleanup_old_tasks() -> int:
|
|||||||
return result.rowcount or 0
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
|
def recover_stalled_task_runs() -> int:
|
||||||
|
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
||||||
|
to 'error'. FC-3i.
|
||||||
|
|
||||||
|
A row gets stuck when the worker dies without emitting
|
||||||
|
task_postrun / task_failure (e.g. OOM, container restart between
|
||||||
|
signals, signal handler raised+logged). Shares the 5-min threshold
|
||||||
|
with recover_interrupted_tasks for consistency.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
update(TaskRun)
|
||||||
|
.where(TaskRun.status == "running")
|
||||||
|
.where(TaskRun.started_at < cutoff)
|
||||||
|
.values(
|
||||||
|
status="error",
|
||||||
|
error_type="RecoverySweep",
|
||||||
|
error_message=(
|
||||||
|
f"no completion signal received within "
|
||||||
|
f"{STUCK_THRESHOLD_MINUTES} min"
|
||||||
|
),
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||||
|
def prune_task_runs() -> dict:
|
||||||
|
"""Daily retention for task_run rows. FC-3i.
|
||||||
|
|
||||||
|
- 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
|
||||||
|
Success is high-volume, not interesting after a day.
|
||||||
|
- 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
|
||||||
|
(7 days default). Failures are operationally interesting longer.
|
||||||
|
- 'running' rows: NEVER deleted by this task. The recovery sweep
|
||||||
|
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||||
|
terminal state; prune doesn't touch in-flight state.
|
||||||
|
- 'retry' rows: treated as failures (>7d).
|
||||||
|
|
||||||
|
Returns dict of how many rows were deleted in each bucket.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||||
|
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
ok_deleted = session.execute(
|
||||||
|
delete(TaskRun)
|
||||||
|
.where(TaskRun.status == "ok")
|
||||||
|
.where(TaskRun.finished_at < ok_cutoff)
|
||||||
|
).rowcount or 0
|
||||||
|
fail_deleted = session.execute(
|
||||||
|
delete(TaskRun)
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||||
|
.where(TaskRun.finished_at < fail_cutoff)
|
||||||
|
).rowcount or 0
|
||||||
|
session.commit()
|
||||||
|
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
||||||
def backfill_phash() -> int:
|
def backfill_phash() -> int:
|
||||||
"""Recompute phash for stored images that have none (imported before
|
"""Recompute phash for stored images that have none (imported before
|
||||||
|
|||||||
@@ -2,9 +2,6 @@
|
|||||||
<Teleport to="body">
|
<Teleport to="body">
|
||||||
<div
|
<div
|
||||||
class="fc-viewer" role="dialog" aria-modal="true"
|
class="fc-viewer" role="dialog" aria-modal="true"
|
||||||
@keydown.esc="$emit('close')"
|
|
||||||
@keydown.left="onArrowLeft"
|
|
||||||
@keydown.right="onArrowRight"
|
|
||||||
tabindex="-1" ref="rootEl"
|
tabindex="-1" ref="rootEl"
|
||||||
>
|
>
|
||||||
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
|
<button class="fc-viewer__close" @click="$emit('close')" aria-label="Close">
|
||||||
@@ -68,7 +65,7 @@ import VideoCanvas from './VideoCanvas.vue'
|
|||||||
import TagPanel from './TagPanel.vue'
|
import TagPanel from './TagPanel.vue'
|
||||||
import ProvenancePanel from './ProvenancePanel.vue'
|
import ProvenancePanel from './ProvenancePanel.vue'
|
||||||
|
|
||||||
defineEmits(['close'])
|
const emit = defineEmits(['close'])
|
||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const rootEl = ref(null)
|
const rootEl = ref(null)
|
||||||
@@ -86,14 +83,39 @@ const integrityBadge = computed(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
let prevBodyOverflow = null
|
let prevBodyOverflow = null
|
||||||
|
|
||||||
|
// Document-level keyboard handler. Bound on document (not on the modal
|
||||||
|
// root) because <video> elements capture focus when the user interacts
|
||||||
|
// with their controls — when video has focus, Escape and arrow keys
|
||||||
|
// don't bubble reliably to ancestors. Listening on document side-steps
|
||||||
|
// that. Filtered via isTextEntry so tag/comment inputs still get their
|
||||||
|
// own keystrokes.
|
||||||
|
function onKeyDown(ev) {
|
||||||
|
if (ev.key === 'Escape') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
emit('close')
|
||||||
|
} else if (ev.key === 'ArrowLeft') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
modal.goPrev()
|
||||||
|
} else if (ev.key === 'ArrowRight') {
|
||||||
|
if (isTextEntry(ev.target)) return
|
||||||
|
ev.preventDefault()
|
||||||
|
modal.goNext()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
prevBodyOverflow = document.body.style.overflow
|
prevBodyOverflow = document.body.style.overflow
|
||||||
document.body.style.overflow = 'hidden'
|
document.body.style.overflow = 'hidden'
|
||||||
|
document.addEventListener('keydown', onKeyDown, true)
|
||||||
await nextFrame()
|
await nextFrame()
|
||||||
rootEl.value?.focus()
|
rootEl.value?.focus()
|
||||||
})
|
})
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
document.body.style.overflow = prevBodyOverflow
|
document.body.style.overflow = prevBodyOverflow
|
||||||
|
document.removeEventListener('keydown', onKeyDown, true)
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => modal.currentImageId, async () => {
|
watch(() => modal.currentImageId, async () => {
|
||||||
@@ -105,16 +127,6 @@ function nextFrame() {
|
|||||||
return new Promise(resolve => requestAnimationFrame(resolve))
|
return new Promise(resolve => requestAnimationFrame(resolve))
|
||||||
}
|
}
|
||||||
|
|
||||||
function onArrowLeft(ev) {
|
|
||||||
if (isTextEntry(ev.target)) return
|
|
||||||
ev.preventDefault()
|
|
||||||
modal.goPrev()
|
|
||||||
}
|
|
||||||
function onArrowRight(ev) {
|
|
||||||
if (isTextEntry(ev.target)) return
|
|
||||||
ev.preventDefault()
|
|
||||||
modal.goNext()
|
|
||||||
}
|
|
||||||
function isTextEntry(el) {
|
function isTextEntry(el) {
|
||||||
if (!el) return false
|
if (!el) return false
|
||||||
const tag = el.tagName
|
const tag = el.tagName
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
<template>
|
||||||
|
<v-table density="compact" class="fc-queues-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th class="text-right">Depth</th>
|
||||||
|
<th class="text-right">Workers</th>
|
||||||
|
<th v-if="!compact" class="text-right">Active</th>
|
||||||
|
<th class="text-right">Last min: ok / err</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="q in QUEUE_NAMES" :key="q">
|
||||||
|
<td>{{ q }}</td>
|
||||||
|
<td class="text-right fc-tabular" :class="depthClass(q)">
|
||||||
|
{{ formatDepth(q) }}
|
||||||
|
</td>
|
||||||
|
<td class="text-right fc-tabular">{{ workerCount(q) }}</td>
|
||||||
|
<td v-if="!compact" class="text-right fc-tabular">{{ activeCount(q) }}</td>
|
||||||
|
<td class="text-right fc-tabular">
|
||||||
|
<span class="fc-ok">{{ recentOk(q) }}</span>
|
||||||
|
<span class="fc-sep">/</span>
|
||||||
|
<span :class="recentErr(q) > 0 ? 'fc-err' : 'fc-muted'">
|
||||||
|
{{ recentErr(q) }}<span v-if="recentErr(q) > 0"> ⚠</span>
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
queues: { type: Object, default: null }, // store.queues
|
||||||
|
workers: { type: Object, default: null }, // store.workers
|
||||||
|
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
|
||||||
|
compact: { type: Boolean, default: false },
|
||||||
|
})
|
||||||
|
|
||||||
|
const QUEUE_NAMES = [
|
||||||
|
'default', 'import', 'thumbnail', 'ml',
|
||||||
|
'download', 'scan', 'maintenance',
|
||||||
|
]
|
||||||
|
|
||||||
|
function formatDepth(name) {
|
||||||
|
const d = props.queues?.queues?.[name]
|
||||||
|
if (d == null) return '—'
|
||||||
|
return d.toLocaleString()
|
||||||
|
}
|
||||||
|
|
||||||
|
function depthClass(name) {
|
||||||
|
const d = props.queues?.queues?.[name]
|
||||||
|
const workers = workerCount(name)
|
||||||
|
// Queue has depth but no workers → warning.
|
||||||
|
if (d != null && d > 0 && workers === 0) return 'fc-warn'
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function workerCount(name) {
|
||||||
|
if (!props.workers?.workers) return 0
|
||||||
|
let count = 0
|
||||||
|
for (const info of Object.values(props.workers.workers)) {
|
||||||
|
if (info.queues?.includes(name)) count++
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
function activeCount(name) {
|
||||||
|
if (!props.workers?.workers) return 0
|
||||||
|
// Active count is per-worker, not per-queue. Approximate: sum of
|
||||||
|
// active_count across workers that subscribe to this queue.
|
||||||
|
let count = 0
|
||||||
|
for (const info of Object.values(props.workers.workers)) {
|
||||||
|
if (info.queues?.includes(name)) count += info.active_count || 0
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
const recentByQueue = computed(() => {
|
||||||
|
const out = {}
|
||||||
|
for (const r of props.recentMinute) {
|
||||||
|
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
||||||
|
if (r.status === 'ok') out[r.queue].ok++
|
||||||
|
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
})
|
||||||
|
|
||||||
|
function recentOk(name) { return recentByQueue.value[name]?.ok || 0 }
|
||||||
|
function recentErr(name) { return recentByQueue.value[name]?.err || 0 }
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-queues-table { background: transparent; }
|
||||||
|
.fc-tabular {
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
font-feature-settings: 'tnum';
|
||||||
|
}
|
||||||
|
.fc-ok { color: rgb(var(--v-theme-success, 76 175 80)); }
|
||||||
|
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||||
|
.fc-warn { color: rgb(var(--v-theme-warning, 255 179 0)); font-weight: 500; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-sep {
|
||||||
|
margin: 0 4px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-activity-summary">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-pulse" size="small" />
|
||||||
|
<span>System activity (last min)</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" rounded="pill"
|
||||||
|
@click="$emit('open-activity')"
|
||||||
|
>
|
||||||
|
View activity
|
||||||
|
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
|
||||||
|
<v-card-text>
|
||||||
|
<v-alert
|
||||||
|
v-if="store.lastError"
|
||||||
|
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>
|
||||||
|
{{ store.lastError }}
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<QueuesTable
|
||||||
|
:queues="store.queues"
|
||||||
|
:workers="store.workers"
|
||||||
|
:recent-minute="store.recentMinute"
|
||||||
|
compact
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
|
defineEmits(['open-activity'])
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
let pollId = null
|
||||||
|
|
||||||
|
function pollOnce() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadQueues()
|
||||||
|
store.loadWorkers()
|
||||||
|
store.loadRecentMinute()
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
pollOnce()
|
||||||
|
pollId = setInterval(pollOnce, 5000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (pollId) { clearInterval(pollId); pollId = null }
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-activity-summary { border-radius: 8px; }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-activity">
|
||||||
|
<!-- Queues + workers pane -->
|
||||||
|
<v-card class="mb-4">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-pulse" size="small" />
|
||||||
|
<span>Queues + workers</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="text-caption fc-muted">
|
||||||
|
updated {{ formatRelative(store.queues?.fetched_at) }}
|
||||||
|
</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<QueuesTable
|
||||||
|
:queues="store.queues"
|
||||||
|
:workers="store.workers"
|
||||||
|
:recent-minute="store.recentMinute"
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<!-- Recent failures pane -->
|
||||||
|
<v-card class="mb-4">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-alert-circle-outline" size="small" />
|
||||||
|
<span>Recent failures (last 24h)</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="text-caption fc-muted">
|
||||||
|
{{ failureCount }} failures
|
||||||
|
</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<div v-if="errorTypes.length" class="mb-3 fc-pills">
|
||||||
|
<v-chip
|
||||||
|
v-for="t in errorTypes" :key="t.name"
|
||||||
|
:color="t.name === filterErrorType ? 'accent' : undefined"
|
||||||
|
size="small" variant="tonal" closable
|
||||||
|
@click="toggleErrorTypeFilter(t.name)"
|
||||||
|
@click:close="toggleErrorTypeFilter(t.name)"
|
||||||
|
>
|
||||||
|
{{ t.name }} × {{ t.count }}
|
||||||
|
</v-chip>
|
||||||
|
</div>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Time</th>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th>Task</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th>Error</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in filteredFailures" :key="r.id">
|
||||||
|
<td class="fc-tabular" :title="r.finished_at">
|
||||||
|
{{ formatRelative(r.finished_at) }}
|
||||||
|
</td>
|
||||||
|
<td>{{ r.queue }}</td>
|
||||||
|
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||||
|
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||||
|
<td class="fc-err" :title="r.error_message">
|
||||||
|
{{ r.error_type }}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!filteredFailures.length">
|
||||||
|
<td colspan="5" class="text-center fc-muted py-4">
|
||||||
|
No failures in the last 24h.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<!-- All activity pane -->
|
||||||
|
<v-card>
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-format-list-bulleted" size="small" />
|
||||||
|
<span>All recent activity</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-select
|
||||||
|
v-model="filterQueue"
|
||||||
|
:items="queueOptions" density="compact" hide-details
|
||||||
|
style="max-width: 180px;"
|
||||||
|
@update:model-value="onFilterChange"
|
||||||
|
/>
|
||||||
|
<v-select
|
||||||
|
v-model="filterStatus"
|
||||||
|
:items="statusOptions" density="compact" hide-details
|
||||||
|
style="max-width: 180px;"
|
||||||
|
@update:model-value="onFilterChange"
|
||||||
|
/>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" rounded="pill"
|
||||||
|
@click="onRefresh"
|
||||||
|
>
|
||||||
|
<v-icon start size="small">mdi-refresh</v-icon>
|
||||||
|
Refresh
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th></th>
|
||||||
|
<th>Time</th>
|
||||||
|
<th>Queue</th>
|
||||||
|
<th>Task</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th class="text-right">Duration</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in store.runs" :key="r.id">
|
||||||
|
<td><v-icon size="small" :color="statusColor(r.status)">{{ statusIcon(r.status) }}</v-icon></td>
|
||||||
|
<td class="fc-tabular" :title="r.started_at">{{ formatRelative(r.started_at) }}</td>
|
||||||
|
<td>{{ r.queue }}</td>
|
||||||
|
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||||
|
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||||
|
<td class="text-right fc-tabular">{{ formatDuration(r.duration_ms) }}</td>
|
||||||
|
<td>{{ r.status }}</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!store.runs.length && !store.loading.runs">
|
||||||
|
<td colspan="7" class="text-center fc-muted py-4">
|
||||||
|
No activity yet.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
<div v-if="store.runsHasMore" class="d-flex justify-center py-3">
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small"
|
||||||
|
:loading="store.loading.runs"
|
||||||
|
@click="onLoadMore"
|
||||||
|
>Load more</v-btn>
|
||||||
|
</div>
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
const filterQueue = ref(null)
|
||||||
|
const filterStatus = ref(null)
|
||||||
|
const filterErrorType = ref(null)
|
||||||
|
|
||||||
|
const queueOptions = [
|
||||||
|
{ title: 'All queues', value: null },
|
||||||
|
{ title: 'import', value: 'import' },
|
||||||
|
{ title: 'thumbnail', value: 'thumbnail' },
|
||||||
|
{ title: 'ml', value: 'ml' },
|
||||||
|
{ title: 'download', value: 'download' },
|
||||||
|
{ title: 'scan', value: 'scan' },
|
||||||
|
{ title: 'maintenance', value: 'maintenance' },
|
||||||
|
{ title: 'default', value: 'default' },
|
||||||
|
]
|
||||||
|
const statusOptions = [
|
||||||
|
{ title: 'All statuses', value: null },
|
||||||
|
{ title: 'Running', value: 'running' },
|
||||||
|
{ title: 'OK', value: 'ok' },
|
||||||
|
{ title: 'Error', value: 'error' },
|
||||||
|
{ title: 'Timeout', value: 'timeout' },
|
||||||
|
{ title: 'Retry', value: 'retry' },
|
||||||
|
]
|
||||||
|
|
||||||
|
let queuesPollId = null
|
||||||
|
let failuresPollId = null
|
||||||
|
|
||||||
|
function pollQueues() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadQueues()
|
||||||
|
store.loadWorkers()
|
||||||
|
store.loadRecentMinute()
|
||||||
|
}
|
||||||
|
function pollFailures() {
|
||||||
|
if (document.hidden) return
|
||||||
|
store.loadFailures()
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
pollQueues()
|
||||||
|
pollFailures()
|
||||||
|
store.setFilter({ queue: null, status: null })
|
||||||
|
store.loadRuns({ reset: true })
|
||||||
|
queuesPollId = setInterval(pollQueues, 3000)
|
||||||
|
failuresPollId = setInterval(pollFailures, 15000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (queuesPollId) clearInterval(queuesPollId)
|
||||||
|
if (failuresPollId) clearInterval(failuresPollId)
|
||||||
|
})
|
||||||
|
|
||||||
|
const errorTypes = computed(() => {
|
||||||
|
const counts = store.failures?.count_by_type || {}
|
||||||
|
return Object.entries(counts).map(([name, count]) => ({ name, count }))
|
||||||
|
})
|
||||||
|
|
||||||
|
const failureCount = computed(() => (store.failures?.recent || []).length)
|
||||||
|
|
||||||
|
const filteredFailures = computed(() => {
|
||||||
|
const all = store.failures?.recent || []
|
||||||
|
if (!filterErrorType.value) return all
|
||||||
|
return all.filter(r => r.error_type === filterErrorType.value)
|
||||||
|
})
|
||||||
|
|
||||||
|
function toggleErrorTypeFilter(name) {
|
||||||
|
filterErrorType.value = filterErrorType.value === name ? null : name
|
||||||
|
if (filterErrorType.value) {
|
||||||
|
// Mirror into the All-activity table too.
|
||||||
|
filterStatus.value = 'error'
|
||||||
|
onFilterChange()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onFilterChange() {
|
||||||
|
store.setFilter({ queue: filterQueue.value, status: filterStatus.value })
|
||||||
|
store.loadRuns({ reset: true })
|
||||||
|
}
|
||||||
|
function onLoadMore() { store.loadRuns({ reset: false }) }
|
||||||
|
function onRefresh() { store.loadRuns({ reset: true }) }
|
||||||
|
|
||||||
|
function statusIcon(status) {
|
||||||
|
return {
|
||||||
|
ok: 'mdi-check-circle',
|
||||||
|
error: 'mdi-close-circle',
|
||||||
|
timeout: 'mdi-clock-alert',
|
||||||
|
retry: 'mdi-refresh',
|
||||||
|
running: 'mdi-timer-sand',
|
||||||
|
}[status] || 'mdi-help-circle'
|
||||||
|
}
|
||||||
|
function statusColor(status) {
|
||||||
|
return {
|
||||||
|
ok: 'success', error: 'error', timeout: 'warning',
|
||||||
|
retry: 'info', running: 'accent',
|
||||||
|
}[status] || 'on-surface-variant'
|
||||||
|
}
|
||||||
|
function shortTaskName(name) {
|
||||||
|
if (!name) return ''
|
||||||
|
const parts = name.split('.')
|
||||||
|
return parts.slice(-1)[0]
|
||||||
|
}
|
||||||
|
function formatDuration(ms) {
|
||||||
|
if (ms == null) return '—'
|
||||||
|
if (ms < 1000) return `${ms} ms`
|
||||||
|
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`
|
||||||
|
return `${(ms / 60_000).toFixed(1)} min`
|
||||||
|
}
|
||||||
|
function formatRelative(iso) {
|
||||||
|
if (!iso) return '—'
|
||||||
|
const then = new Date(iso).getTime()
|
||||||
|
const now = Date.now()
|
||||||
|
const diff = Math.max(0, (now - then) / 1000)
|
||||||
|
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||||
|
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||||
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||||
|
return `${Math.floor(diff / 86400)}d ago`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-activity { padding: 0; }
|
||||||
|
.fc-pills {
|
||||||
|
display: flex; gap: 6px; flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
.fc-tabular {
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
font-feature-settings: 'tnum';
|
||||||
|
}
|
||||||
|
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||||
|
const api = useApi()
|
||||||
|
|
||||||
|
// Live polled state.
|
||||||
|
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
|
||||||
|
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
|
||||||
|
const recentMinute = ref([]) // last-60s rows (for Overview summary)
|
||||||
|
const failures = ref(null) // { recent, count_by_type, since }
|
||||||
|
|
||||||
|
// Paginated runs (Activity tab "All recent activity" pane).
|
||||||
|
const runs = ref([])
|
||||||
|
const runsCursor = ref(null)
|
||||||
|
const runsHasMore = ref(false)
|
||||||
|
const runsFilter = ref({ queue: null, status: null, limit: 50 })
|
||||||
|
|
||||||
|
const loading = ref({ queues: false, workers: false, runs: false, failures: false })
|
||||||
|
const lastError = ref(null)
|
||||||
|
|
||||||
|
async function loadQueues() {
|
||||||
|
loading.value.queues = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
queues.value = await api.get('/api/system/activity/queues')
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value.queues = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadWorkers() {
|
||||||
|
loading.value.workers = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
workers.value = await api.get('/api/system/activity/workers')
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value.workers = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadRecentMinute() {
|
||||||
|
// Used by the Overview summary card: pull last 60s of runs to compute
|
||||||
|
// per-queue ok/err counts. One call covers all queues; UI groups.
|
||||||
|
try {
|
||||||
|
const body = await api.get('/api/system/activity/runs', {
|
||||||
|
params: { limit: 200 },
|
||||||
|
})
|
||||||
|
recentMinute.value = body.runs || []
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadRuns({ reset = false } = {}) {
|
||||||
|
loading.value.runs = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
const params = { limit: runsFilter.value.limit }
|
||||||
|
if (runsFilter.value.queue) params.queue = runsFilter.value.queue
|
||||||
|
if (runsFilter.value.status) params.status = runsFilter.value.status
|
||||||
|
if (!reset && runsCursor.value) params.before_id = runsCursor.value
|
||||||
|
const body = await api.get('/api/system/activity/runs', { params })
|
||||||
|
runs.value = reset ? body.runs : [...runs.value, ...body.runs]
|
||||||
|
runsCursor.value = body.next_cursor
|
||||||
|
runsHasMore.value = body.next_cursor !== null
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value.runs = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFailures() {
|
||||||
|
loading.value.failures = true
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
failures.value = await api.get('/api/system/activity/failures')
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
} finally {
|
||||||
|
loading.value.failures = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setFilter(filter) {
|
||||||
|
runsFilter.value = { ...runsFilter.value, ...filter }
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
queues, workers, recentMinute, failures,
|
||||||
|
runs, runsCursor, runsHasMore, runsFilter,
|
||||||
|
loading, lastError,
|
||||||
|
loadQueues, loadWorkers, loadRecentMinute,
|
||||||
|
loadRuns, loadFailures, setFilter,
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
<v-container fluid class="py-6">
|
<v-container fluid class="py-6">
|
||||||
<v-tabs v-model="tab" color="accent" class="mb-4">
|
<v-tabs v-model="tab" color="accent" class="mb-4">
|
||||||
<v-tab value="overview">Overview</v-tab>
|
<v-tab value="overview">Overview</v-tab>
|
||||||
|
<v-tab value="activity">Activity</v-tab>
|
||||||
<v-tab value="import">Import</v-tab>
|
<v-tab value="import">Import</v-tab>
|
||||||
<v-tab value="maintenance">Maintenance</v-tab>
|
<v-tab value="maintenance">Maintenance</v-tab>
|
||||||
</v-tabs>
|
</v-tabs>
|
||||||
@@ -9,6 +10,10 @@
|
|||||||
<v-window v-model="tab">
|
<v-window v-model="tab">
|
||||||
<v-window-item value="overview">
|
<v-window-item value="overview">
|
||||||
<SystemStatsCards :stats="system.stats" />
|
<SystemStatsCards :stats="system.stats" />
|
||||||
|
<SystemActivitySummary
|
||||||
|
class="mt-4"
|
||||||
|
@open-activity="tab = 'activity'"
|
||||||
|
/>
|
||||||
<v-alert
|
<v-alert
|
||||||
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
|
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
|
||||||
type="info" variant="tonal" class="mt-4" closable
|
type="info" variant="tonal" class="mt-4" closable
|
||||||
@@ -18,6 +23,10 @@
|
|||||||
</v-alert>
|
</v-alert>
|
||||||
</v-window-item>
|
</v-window-item>
|
||||||
|
|
||||||
|
<v-window-item value="activity">
|
||||||
|
<SystemActivityTab />
|
||||||
|
</v-window-item>
|
||||||
|
|
||||||
<v-window-item value="import">
|
<v-window-item value="import">
|
||||||
<ImportTriggerPanel />
|
<ImportTriggerPanel />
|
||||||
<v-divider class="my-6" />
|
<v-divider class="my-6" />
|
||||||
@@ -38,6 +47,8 @@ import { onMounted, onUnmounted, ref, watch } from 'vue'
|
|||||||
import { useSystemStore } from '../stores/system.js'
|
import { useSystemStore } from '../stores/system.js'
|
||||||
import { useImportStore } from '../stores/import.js'
|
import { useImportStore } from '../stores/import.js'
|
||||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||||
|
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||||
|
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||||
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
||||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||||
|
|||||||
@@ -83,24 +83,11 @@ async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
|
|||||||
assert resp.status_code == 202
|
assert resp.status_code == 202
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
# Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
|
||||||
async def test_post_apply_without_backup_rejected(client, monkeypatch):
|
# the migration backup-gate rejected applies without a recent backup.
|
||||||
monkeypatch.setattr(
|
# That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
|
||||||
"backend.app.api.migrate._has_recent_pre_migration_backup",
|
# FC-3h will own backup as a first-class feature) so the test was
|
||||||
lambda: False,
|
# testing dead behavior. Removed rather than rewritten.
|
||||||
)
|
|
||||||
export = {
|
|
||||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
|
||||||
"subscriptions": [], "credentials": [],
|
|
||||||
}
|
|
||||||
resp = await client.post(
|
|
||||||
"/api/migrate/gs_ingest",
|
|
||||||
form={"dry_run": "false"},
|
|
||||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
|
||||||
)
|
|
||||||
assert resp.status_code == 400
|
|
||||||
body = await resp.get_json()
|
|
||||||
assert body["error"] == "no_backup"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -0,0 +1,267 @@
|
|||||||
|
"""FC-3i: /api/system/activity/* endpoint tests.
|
||||||
|
|
||||||
|
Mocks Redis LLEN + celery inspect via monkeypatch on the module-level
|
||||||
|
_read_queues_sync / _read_workers_sync helpers, so tests don't need
|
||||||
|
a live broker or running workers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
|
||||||
|
from backend.app import create_app
|
||||||
|
from backend.app.api import system_activity as activity_module
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def app():
|
||||||
|
return create_app()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def client(app):
|
||||||
|
async with app.test_client() as c:
|
||||||
|
yield c
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_caches(monkeypatch):
|
||||||
|
"""Clear the module-level caches between tests so cache state from
|
||||||
|
one test doesn't leak into the next."""
|
||||||
|
monkeypatch.setitem(activity_module._QUEUE_CACHE, "data", None)
|
||||||
|
monkeypatch.setitem(activity_module._QUEUE_CACHE, "ts", 0.0)
|
||||||
|
monkeypatch.setitem(activity_module._WORKER_CACHE, "data", None)
|
||||||
|
monkeypatch.setitem(activity_module._WORKER_CACHE, "ts", 0.0)
|
||||||
|
|
||||||
|
|
||||||
|
# --- /queues -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_queues_returns_all_known_queues(client, monkeypatch):
|
||||||
|
def _fake():
|
||||||
|
return {
|
||||||
|
"queues": dict.fromkeys(activity_module._QUEUE_NAMES, 0),
|
||||||
|
"fetched_at": datetime.now(UTC).isoformat(),
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(activity_module, "_read_queues_sync", _fake)
|
||||||
|
|
||||||
|
resp = await client.get("/api/system/activity/queues")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert set(body["queues"].keys()) == set(activity_module._QUEUE_NAMES)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_queues_cached_within_ttl(client, monkeypatch):
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def _counting():
|
||||||
|
call_count["n"] += 1
|
||||||
|
return {"queues": {}, "fetched_at": "x"}
|
||||||
|
monkeypatch.setattr(activity_module, "_read_queues_sync", _counting)
|
||||||
|
|
||||||
|
await client.get("/api/system/activity/queues")
|
||||||
|
await client.get("/api/system/activity/queues")
|
||||||
|
await client.get("/api/system/activity/queues")
|
||||||
|
assert call_count["n"] == 1 # cached after the first call
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_queues_redis_failure_returns_null_per_failing_queue(client, monkeypatch):
|
||||||
|
"""_read_queues_sync's per-queue try/except returns None on failure."""
|
||||||
|
def _partial():
|
||||||
|
out = dict.fromkeys(activity_module._QUEUE_NAMES, 0)
|
||||||
|
out["ml"] = None # simulate broker hiccup on the ml queue
|
||||||
|
return {"queues": out, "fetched_at": "x"}
|
||||||
|
monkeypatch.setattr(activity_module, "_read_queues_sync", _partial)
|
||||||
|
|
||||||
|
resp = await client.get("/api/system/activity/queues")
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["queues"]["ml"] is None
|
||||||
|
assert body["queues"]["import"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
# --- /workers ------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_workers_returns_per_worker_queue_membership(client, monkeypatch):
|
||||||
|
def _fake():
|
||||||
|
return {
|
||||||
|
"workers": {
|
||||||
|
"worker@host1": {"queues": ["import", "thumbnail"], "active_count": 2},
|
||||||
|
"worker@host2": {"queues": ["ml"], "active_count": 0},
|
||||||
|
},
|
||||||
|
"fetched_at": "x",
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(activity_module, "_read_workers_sync", _fake)
|
||||||
|
|
||||||
|
resp = await client.get("/api/system/activity/workers")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert "worker@host1" in body["workers"]
|
||||||
|
assert "import" in body["workers"]["worker@host1"]["queues"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_workers_cached_within_ttl(client, monkeypatch):
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def _counting():
|
||||||
|
call_count["n"] += 1
|
||||||
|
return {"workers": {}, "fetched_at": "x"}
|
||||||
|
monkeypatch.setattr(activity_module, "_read_workers_sync", _counting)
|
||||||
|
|
||||||
|
await client.get("/api/system/activity/workers")
|
||||||
|
await client.get("/api/system/activity/workers")
|
||||||
|
assert call_count["n"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
# --- /runs ---------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def _seed_runs(db):
|
||||||
|
"""Insert 5 task_run rows for paging tests."""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
for i in range(5):
|
||||||
|
db.add(TaskRun(
|
||||||
|
celery_task_id=f"tid-{i}",
|
||||||
|
queue="import" if i % 2 == 0 else "ml",
|
||||||
|
task_name=f"backend.app.tasks.fake.task_{i}",
|
||||||
|
target_id=i,
|
||||||
|
started_at=now - timedelta(seconds=10 - i),
|
||||||
|
finished_at=now - timedelta(seconds=9 - i),
|
||||||
|
duration_ms=1000,
|
||||||
|
status="ok" if i < 3 else "error",
|
||||||
|
error_type="ValueError" if i >= 3 else None,
|
||||||
|
error_message="boom" if i >= 3 else None,
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_paginated_descending_by_id(client, _seed_runs):
|
||||||
|
resp = await client.get("/api/system/activity/runs?limit=3")
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert len(body["runs"]) == 3
|
||||||
|
# Descending: latest id first.
|
||||||
|
ids = [r["id"] for r in body["runs"]]
|
||||||
|
assert ids == sorted(ids, reverse=True)
|
||||||
|
assert body["next_cursor"] is not None # 5 total, asked for 3 → more
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_filter_by_queue(client, _seed_runs):
|
||||||
|
resp = await client.get("/api/system/activity/runs?queue=ml")
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert all(r["queue"] == "ml" for r in body["runs"])
|
||||||
|
assert len(body["runs"]) == 2 # i=1, i=3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_filter_by_status(client, _seed_runs):
|
||||||
|
resp = await client.get("/api/system/activity/runs?status=error")
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert all(r["status"] == "error" for r in body["runs"])
|
||||||
|
assert len(body["runs"]) == 2 # i=3, i=4
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_keyset_cursor(client, _seed_runs):
|
||||||
|
page1 = await (await client.get("/api/system/activity/runs?limit=2")).get_json()
|
||||||
|
assert len(page1["runs"]) == 2
|
||||||
|
assert page1["next_cursor"] is not None
|
||||||
|
|
||||||
|
page2 = await (await client.get(
|
||||||
|
f"/api/system/activity/runs?limit=2&before_id={page1['next_cursor']}"
|
||||||
|
)).get_json()
|
||||||
|
assert len(page2["runs"]) == 2
|
||||||
|
page1_ids = {r["id"] for r in page1["runs"]}
|
||||||
|
page2_ids = {r["id"] for r in page2["runs"]}
|
||||||
|
assert page1_ids.isdisjoint(page2_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_invalid_limit_400(client):
|
||||||
|
resp = await client.get("/api/system/activity/runs?limit=not-a-number")
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
# --- /failures -----------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def _seed_failures(db):
|
||||||
|
"""Mix of ok / error / timeout rows; some old, some recent."""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
recent = [
|
||||||
|
("error", "OperationalError"),
|
||||||
|
("error", "OperationalError"),
|
||||||
|
("error", "OperationalError"),
|
||||||
|
("timeout", "TimeoutError"),
|
||||||
|
("timeout", "TimeoutError"),
|
||||||
|
("error", "OSError"),
|
||||||
|
("ok", None), # success — should NOT appear in failures
|
||||||
|
]
|
||||||
|
for i, (status, etype) in enumerate(recent):
|
||||||
|
db.add(TaskRun(
|
||||||
|
celery_task_id=f"f-{i}",
|
||||||
|
queue="ml",
|
||||||
|
task_name="backend.app.tasks.fake.x",
|
||||||
|
target_id=i,
|
||||||
|
started_at=now - timedelta(seconds=120 - i),
|
||||||
|
finished_at=now - timedelta(seconds=60 - i),
|
||||||
|
duration_ms=1000,
|
||||||
|
status=status,
|
||||||
|
error_type=etype,
|
||||||
|
error_message="boom" if status != "ok" else None,
|
||||||
|
))
|
||||||
|
# One ancient failure (>24h) that should NOT appear.
|
||||||
|
db.add(TaskRun(
|
||||||
|
celery_task_id="f-old",
|
||||||
|
queue="ml",
|
||||||
|
task_name="backend.app.tasks.fake.x",
|
||||||
|
target_id=999,
|
||||||
|
started_at=now - timedelta(hours=30),
|
||||||
|
finished_at=now - timedelta(hours=29),
|
||||||
|
duration_ms=1000,
|
||||||
|
status="error",
|
||||||
|
error_type="OldError",
|
||||||
|
error_message="ancient",
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_failures_returns_recent_errors_and_timeouts(client, _seed_failures):
|
||||||
|
resp = await client.get("/api/system/activity/failures")
|
||||||
|
body = await resp.get_json()
|
||||||
|
statuses = {r["status"] for r in body["recent"]}
|
||||||
|
assert statuses == {"error", "timeout"}
|
||||||
|
# No ok rows.
|
||||||
|
assert "ok" not in statuses
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_failures_count_by_type_groups_correctly(client, _seed_failures):
|
||||||
|
resp = await client.get("/api/system/activity/failures")
|
||||||
|
body = await resp.get_json()
|
||||||
|
counts = body["count_by_type"]
|
||||||
|
assert counts.get("OperationalError") == 3
|
||||||
|
assert counts.get("TimeoutError") == 2
|
||||||
|
assert counts.get("OSError") == 1
|
||||||
|
assert "OldError" not in counts # outside 24h window
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_failures_only_within_24h_window(client, _seed_failures):
|
||||||
|
resp = await client.get("/api/system/activity/failures")
|
||||||
|
body = await resp.get_json()
|
||||||
|
ids = {r["error_type"] for r in body["recent"]}
|
||||||
|
assert "OldError" not in ids
|
||||||
@@ -93,3 +93,174 @@ def test_cleanup_old_deletes_finished_old(db_sync):
|
|||||||
|
|
||||||
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
|
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
|
||||||
assert remaining == {"/import/b.jpg", "/import/c.jpg"}
|
assert remaining == {"/import/b.jpg", "/import/c.jpg"}
|
||||||
|
|
||||||
|
|
||||||
|
# --- FC-3i: task_run sweep + retention -----------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
|
||||||
|
error_type=None):
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
row = TaskRun(
|
||||||
|
celery_task_id="x",
|
||||||
|
queue="ml",
|
||||||
|
task_name="backend.app.tasks.fake.t",
|
||||||
|
target_id=1,
|
||||||
|
started_at=started_at,
|
||||||
|
finished_at=finished_at,
|
||||||
|
duration_ms=1000 if finished_at else None,
|
||||||
|
status=status,
|
||||||
|
error_type=error_type,
|
||||||
|
error_message="x" if status in ("error", "timeout") else None,
|
||||||
|
)
|
||||||
|
db_sync.add(row)
|
||||||
|
db_sync.flush()
|
||||||
|
return row.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_stalled_task_runs_flips_old_running_to_error(db_sync):
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
stale_id = _make_task_run(
|
||||||
|
db_sync, status="running", started_at=now - timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
recovered = recover_stalled_task_runs.apply().get()
|
||||||
|
assert recovered == 1
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.id == stale_id)
|
||||||
|
).scalar_one()
|
||||||
|
error_type = db_sync.execute(
|
||||||
|
select(TaskRun.error_type).where(TaskRun.id == stale_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert status == "error"
|
||||||
|
assert error_type == "RecoverySweep"
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
fresh_id = _make_task_run(
|
||||||
|
db_sync, status="running", started_at=now - timedelta(seconds=30),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
recovered = recover_stalled_task_runs.apply().get()
|
||||||
|
assert recovered == 0
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.id == fresh_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert status == "running"
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import prune_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
old_id = _make_task_run(
|
||||||
|
db_sync, status="ok",
|
||||||
|
started_at=now - timedelta(hours=30),
|
||||||
|
finished_at=now - timedelta(hours=29),
|
||||||
|
)
|
||||||
|
recent_id = _make_task_run(
|
||||||
|
db_sync, status="ok",
|
||||||
|
started_at=now - timedelta(hours=2),
|
||||||
|
finished_at=now - timedelta(hours=1),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
result = prune_task_runs.apply().get()
|
||||||
|
assert result["ok_deleted"] == 1
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
surviving_ids = set(db_sync.execute(
|
||||||
|
select(TaskRun.id).where(TaskRun.id.in_([old_id, recent_id]))
|
||||||
|
).scalars().all())
|
||||||
|
assert surviving_ids == {recent_id}
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_task_runs_deletes_failures_older_than_7d(db_sync):
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import prune_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
old_id = _make_task_run(
|
||||||
|
db_sync, status="error", error_type="OldError",
|
||||||
|
started_at=now - timedelta(days=10),
|
||||||
|
finished_at=now - timedelta(days=9),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
result = prune_task_runs.apply().get()
|
||||||
|
assert result["failures_deleted"] >= 1
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
surviving = db_sync.execute(
|
||||||
|
select(TaskRun.id).where(TaskRun.id == old_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
assert surviving is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_task_runs_keeps_recent_failures(db_sync):
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import prune_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
recent_id = _make_task_run(
|
||||||
|
db_sync, status="error", error_type="RecentError",
|
||||||
|
started_at=now - timedelta(days=6),
|
||||||
|
finished_at=now - timedelta(days=5),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
prune_task_runs.apply().get()
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
surviving = db_sync.execute(
|
||||||
|
select(TaskRun.id).where(TaskRun.id == recent_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
assert surviving == recent_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_task_runs_never_deletes_running(db_sync):
|
||||||
|
"""Even a 30-day-old running row stays — recovery sweep is the
|
||||||
|
mechanism that flips them; prune doesn't touch in-flight."""
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import prune_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
ancient_id = _make_task_run(
|
||||||
|
db_sync, status="running",
|
||||||
|
started_at=now - timedelta(days=30),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
prune_task_runs.apply().get()
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
surviving = db_sync.execute(
|
||||||
|
select(TaskRun.id).where(TaskRun.id == ancient_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
assert surviving == ancient_id
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
"""FC-3i: Celery signal → task_run row tests.
|
||||||
|
|
||||||
|
Uses task_always_eager so signals fire synchronously in-process during
|
||||||
|
the test. Scoped autouse fixture keeps the eager mode confined to this
|
||||||
|
file (other tests may break with eager mode enabled globally).
|
||||||
|
|
||||||
|
Test assertions on signal-mutated rows use COLUMN SELECTS / db_sync.
|
||||||
|
scalar (per reference_async_coredml_test_assertions). ORM attribute
|
||||||
|
access on a row a signal handler just mutated risks MissingGreenlet
|
||||||
|
across async/sync boundaries.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.celery_app import celery
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _eager_celery(monkeypatch):
|
||||||
|
"""task_always_eager so signals fire in-process during the test."""
|
||||||
|
monkeypatch.setattr(celery.conf, "task_always_eager", True)
|
||||||
|
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_prerun_inserts_running_row(db_sync):
|
||||||
|
"""Fire a no-op task; assert task_run row inserted with running status."""
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
generate_thumbnail.delay(999_999) # nonexistent image_id → no-op
|
||||||
|
|
||||||
|
rows = db_sync.execute(
|
||||||
|
select(TaskRun.queue, TaskRun.target_id, TaskRun.task_name)
|
||||||
|
.where(TaskRun.target_id == 999_999)
|
||||||
|
).all()
|
||||||
|
assert len(rows) == 1
|
||||||
|
queue, target_id, task_name = rows[0]
|
||||||
|
assert queue == "thumbnail"
|
||||||
|
assert target_id == 999_999
|
||||||
|
assert task_name.endswith(".generate_thumbnail")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_postrun_marks_ok_with_duration(db_sync):
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
generate_thumbnail.delay(999_998)
|
||||||
|
|
||||||
|
row = db_sync.execute(
|
||||||
|
select(
|
||||||
|
TaskRun.status, TaskRun.finished_at, TaskRun.duration_ms,
|
||||||
|
).where(TaskRun.target_id == 999_998)
|
||||||
|
).one()
|
||||||
|
status, finished_at, duration_ms = row
|
||||||
|
assert status == "ok"
|
||||||
|
assert finished_at is not None
|
||||||
|
assert duration_ms is not None and duration_ms >= 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_failure_records_error_type_and_message(db_sync, monkeypatch):
|
||||||
|
"""Force a tracked task to raise; assert error fields populated."""
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
|
||||||
|
def _explode(*a, **k):
|
||||||
|
raise ValueError("synthetic failure")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.thumbnail.generate_thumbnail.run",
|
||||||
|
_explode,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
generate_thumbnail.delay(999_997).get(propagate=True)
|
||||||
|
|
||||||
|
row = db_sync.execute(
|
||||||
|
select(
|
||||||
|
TaskRun.status, TaskRun.error_type, TaskRun.error_message,
|
||||||
|
TaskRun.finished_at,
|
||||||
|
).where(TaskRun.target_id == 999_997)
|
||||||
|
.order_by(TaskRun.id.desc()).limit(1)
|
||||||
|
).one()
|
||||||
|
status, error_type, error_message, finished_at = row
|
||||||
|
assert status == "error"
|
||||||
|
assert error_type == "ValueError"
|
||||||
|
assert "synthetic failure" in (error_message or "")
|
||||||
|
assert finished_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_soft_time_limit_records_timeout_status(db_sync, monkeypatch):
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
|
||||||
|
def _timeout(*a, **k):
|
||||||
|
raise SoftTimeLimitExceeded("simulated soft limit")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.thumbnail.generate_thumbnail.run",
|
||||||
|
_timeout,
|
||||||
|
)
|
||||||
|
with pytest.raises(SoftTimeLimitExceeded):
|
||||||
|
generate_thumbnail.delay(999_996).get(propagate=True)
|
||||||
|
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.target_id == 999_996)
|
||||||
|
.order_by(TaskRun.id.desc()).limit(1)
|
||||||
|
).scalar_one()
|
||||||
|
assert status == "timeout"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_untracked_celery_internal_tasks_skip(db_sync):
|
||||||
|
"""celery.chord_unlock is in the untracked list; firing it should
|
||||||
|
NOT insert a task_run row."""
|
||||||
|
before = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
|
||||||
|
# Send a chord_unlock-style task via apply (no real task to dispatch).
|
||||||
|
# The signal handler checks task.name against _UNTRACKED_TASK_NAMES;
|
||||||
|
# if name is in the list, no row is inserted. We verify by firing
|
||||||
|
# a regular task and asserting only ONE row appears, not two.
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
generate_thumbnail.delay(999_995)
|
||||||
|
after = db_sync.execute(select(func.count(TaskRun.id))).scalar_one()
|
||||||
|
assert after - before == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_args_summary_truncated_to_255(db_sync, monkeypatch):
|
||||||
|
"""Task with a very large repr(args); assert args_summary <= 255."""
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
|
||||||
|
# Force a giant arg by patching repr(args) target — actually just
|
||||||
|
# call with a large int that produces a long repr.
|
||||||
|
big_id = int("9" * 200) # 200-char repr
|
||||||
|
# The signal handler reads repr(args), so this naturally produces
|
||||||
|
# a long args_summary that should get truncated to 255.
|
||||||
|
try:
|
||||||
|
generate_thumbnail.delay(big_id).get(timeout=5, propagate=False)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
args_summary = db_sync.execute(
|
||||||
|
select(TaskRun.args_summary)
|
||||||
|
.where(TaskRun.task_name.endswith(".generate_thumbnail"))
|
||||||
|
.order_by(TaskRun.id.desc()).limit(1)
|
||||||
|
).scalar_one()
|
||||||
|
assert args_summary is None or len(args_summary) <= 255
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_error_message_truncated_to_2000(db_sync, monkeypatch):
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
|
||||||
|
huge_message = "X" * 5000
|
||||||
|
|
||||||
|
def _raise_huge(*a, **k):
|
||||||
|
raise RuntimeError(huge_message)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.thumbnail.generate_thumbnail.run", _raise_huge,
|
||||||
|
)
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
generate_thumbnail.delay(999_994).get(propagate=True)
|
||||||
|
|
||||||
|
error_message = db_sync.execute(
|
||||||
|
select(TaskRun.error_message).where(TaskRun.target_id == 999_994)
|
||||||
|
.order_by(TaskRun.id.desc()).limit(1)
|
||||||
|
).scalar_one()
|
||||||
|
assert error_message is not None
|
||||||
|
assert len(error_message) <= 2000
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_signal_handler_failure_does_not_break_task(db_sync, monkeypatch):
|
||||||
|
"""Monkeypatch sync_session_factory in the signals module to raise;
|
||||||
|
assert the task still returns its result and the worker doesn't
|
||||||
|
propagate the signal-handler exception."""
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
|
||||||
|
def _broken_factory():
|
||||||
|
raise RuntimeError("db down")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.celery_signals.sync_session_factory", _broken_factory,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Task should still execute (no-op on missing image) without raising.
|
||||||
|
result = generate_thumbnail.delay(999_993).get(timeout=5)
|
||||||
|
assert result is not None # task ran; signal handler errors swallowed
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("args,expected", [
|
||||||
|
((42,), 42),
|
||||||
|
((42, "ignored"), 42),
|
||||||
|
(("123",), 123),
|
||||||
|
(("not-int",), None),
|
||||||
|
((), None),
|
||||||
|
(None, None),
|
||||||
|
])
|
||||||
|
def test_target_id_extracted_from_first_int_arg(args, expected):
|
||||||
|
"""Unit test the helper directly; no DB needed."""
|
||||||
|
from backend.app.celery_signals import _target_id_from_args
|
||||||
|
assert _target_id_from_args(args) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_finalize_sets_retry_status_with_retry_count(db_sync):
|
||||||
|
"""Direct unit-shape test of _finalize's retry path. We can't
|
||||||
|
reliably trigger Celery's task_retry signal under eager mode with
|
||||||
|
monkeypatched .run (the bind=True 'self' injection breaks), so
|
||||||
|
test the finalize helper directly — that's the code the retry
|
||||||
|
signal handler delegates to anyway."""
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from backend.app.celery_signals import _finalize
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
|
||||||
|
# Seed a 'running' row that _finalize can target.
|
||||||
|
row = TaskRun(
|
||||||
|
celery_task_id="retry-test-tid",
|
||||||
|
queue="ml",
|
||||||
|
task_name="backend.app.tasks.fake.t",
|
||||||
|
target_id=999_992,
|
||||||
|
started_at=datetime.now(UTC),
|
||||||
|
status="running",
|
||||||
|
)
|
||||||
|
db_sync.add(row)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
_finalize(
|
||||||
|
"retry-test-tid", status="retry",
|
||||||
|
error_type="RuntimeError",
|
||||||
|
error_message="first attempt",
|
||||||
|
retry_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Column-select assertions — _finalize mutated the row via its own
|
||||||
|
# session; re-read fresh state through db_sync.
|
||||||
|
db_sync.expire_all()
|
||||||
|
status, retry_count, error_type = db_sync.execute(
|
||||||
|
select(TaskRun.status, TaskRun.retry_count, TaskRun.error_type)
|
||||||
|
.where(TaskRun.celery_task_id == "retry-test-tid")
|
||||||
|
).one()
|
||||||
|
assert status == "retry"
|
||||||
|
assert retry_count == 1
|
||||||
|
assert error_type == "RuntimeError"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_worker_hostname_recorded(db_sync):
|
||||||
|
"""Sender's hostname should land in worker_hostname."""
|
||||||
|
from backend.app.tasks.thumbnail import generate_thumbnail
|
||||||
|
generate_thumbnail.delay(999_991)
|
||||||
|
|
||||||
|
hostname = db_sync.execute(
|
||||||
|
select(TaskRun.worker_hostname).where(TaskRun.target_id == 999_991)
|
||||||
|
).scalar_one()
|
||||||
|
# Under eager mode hostname may be None or "celery@local"; just
|
||||||
|
# assert the column is touched (either string or None — never throws).
|
||||||
|
assert hostname is None or isinstance(hostname, str)
|
||||||
Reference in New Issue
Block a user