Compare commits
8 Commits
ext-1.0.5
...
v26.05.28.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 319e8c1d18 | |||
| dcfe55d731 | |||
| e3cdd0f92b | |||
| e77afe8295 | |||
| 57a22d6098 | |||
| a85880f965 | |||
| 407de18ff6 | |||
| b1b129ce9f |
@@ -0,0 +1,53 @@
|
|||||||
|
"""import_task.recovery_count + refetched — poison-pill circuit breaker
|
||||||
|
|
||||||
|
Revision ID: 0026
|
||||||
|
Revises: 0025
|
||||||
|
Create Date: 2026-05-28
|
||||||
|
|
||||||
|
Backs the import-task resilience work (operator-flagged 2026-05-28):
|
||||||
|
|
||||||
|
- recovery_count: how many times recover_interrupted_tasks has
|
||||||
|
re-queued this row from a stuck 'processing' state. A row that
|
||||||
|
hard-crashes the worker (OOM / segfault on a corrupt or oversized
|
||||||
|
input) leaves no terminal flip, so the sweep re-queues it — and
|
||||||
|
without a cap it would loop forever, re-crashing the worker each
|
||||||
|
time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a
|
||||||
|
diagnostic instead.
|
||||||
|
|
||||||
|
- refetched: whether a one-shot re-download has already been attempted
|
||||||
|
for this task's file. Bounds the Layer-2 re-fetch remediation to a
|
||||||
|
single attempt so source-side corruption doesn't loop.
|
||||||
|
|
||||||
|
Both default to 0 / false; additive, no backfill needed.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0026"
|
||||||
|
down_revision: Union[str, None] = "0025"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"import_task",
|
||||||
|
sa.Column(
|
||||||
|
"recovery_count", sa.Integer(), nullable=False,
|
||||||
|
server_default="0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_task",
|
||||||
|
sa.Column(
|
||||||
|
"refetched", sa.Boolean(), nullable=False,
|
||||||
|
server_default=sa.false(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_task", "refetched")
|
||||||
|
op.drop_column("import_task", "recovery_count")
|
||||||
@@ -114,18 +114,55 @@ async def retry_failed():
|
|||||||
status="queued", error=None,
|
status="queued", error=None,
|
||||||
started_at=None, finished_at=None,
|
started_at=None, finished_at=None,
|
||||||
)
|
)
|
||||||
.returning(ImportTask.id)
|
.returning(ImportTask.id, ImportTask.task_type)
|
||||||
)
|
)
|
||||||
failed_ids = [row[0] for row in result.all()]
|
failed = result.all()
|
||||||
if not failed_ids:
|
if not failed:
|
||||||
return jsonify({"retried": 0})
|
return jsonify({"retried": 0})
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
from ..tasks.import_file import import_media_file
|
from ..tasks.import_file import enqueue_import
|
||||||
for tid in failed_ids:
|
for tid, task_type in failed:
|
||||||
import_media_file.delay(tid)
|
enqueue_import(tid, task_type)
|
||||||
|
|
||||||
return jsonify({"retried": len(failed_ids)})
|
return jsonify({"retried": len(failed)})
|
||||||
|
|
||||||
|
|
||||||
|
@import_admin_bp.route("/tasks/<int:task_id>/refetch", methods=["POST"])
|
||||||
|
async def refetch_task(task_id: int):
|
||||||
|
"""Layer-2 one-shot re-download: delete the (corrupt) file behind a
|
||||||
|
failed import task and re-run its source's downloader to fetch a
|
||||||
|
fresh copy. Only works for files that resolve to an enabled,
|
||||||
|
real-URL subscription Source; filesystem-only imports return
|
||||||
|
no_source.
|
||||||
|
|
||||||
|
Returns one of: refetch_queued (+source_id) / no_source /
|
||||||
|
already_refetched / not_found / not_failed.
|
||||||
|
"""
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(_refetch_task_sync, task_id)
|
||||||
|
if result["status"] == "not_found":
|
||||||
|
return jsonify(result), 404
|
||||||
|
if result["status"] == "not_failed":
|
||||||
|
return jsonify(result), 400
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _refetch_task_sync(session, task_id: int) -> dict:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..models import ImportSettings
|
||||||
|
from ..services.refetch_service import attempt_refetch
|
||||||
|
|
||||||
|
task = session.get(ImportTask, task_id)
|
||||||
|
if task is None:
|
||||||
|
return {"status": "not_found"}
|
||||||
|
if task.status != "failed":
|
||||||
|
return {"status": "not_failed"}
|
||||||
|
settings = session.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
return attempt_refetch(session, task, Path(settings.import_scan_path))
|
||||||
|
|
||||||
|
|
||||||
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||||
|
|||||||
@@ -8,7 +8,16 @@ been processing longer than the stuck-task threshold.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
from sqlalchemy import (
|
||||||
|
BigInteger,
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -26,6 +35,13 @@ class ImportTask(Base):
|
|||||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||||
|
|
||||||
|
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
||||||
|
# how many times the stuck-task sweep has re-queued this row; after
|
||||||
|
# the cap it's failed with a diagnostic instead of looping. refetched
|
||||||
|
# bounds the one-shot re-download remediation to a single attempt.
|
||||||
|
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
|
||||||
result_image_id: Mapped[int | None] = mapped_column(
|
result_image_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ from ..models import (
|
|||||||
PostAttachment,
|
PostAttachment,
|
||||||
Source,
|
Source,
|
||||||
)
|
)
|
||||||
|
from ..utils import safe_probe
|
||||||
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
||||||
from ..utils.phash import compute_phash, find_similar
|
from ..utils.phash import compute_phash, find_similar
|
||||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||||
@@ -407,6 +408,29 @@ class Importer:
|
|||||||
return ImportResult(status="attached")
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
def _import_archive(self, source: Path) -> ImportResult:
|
def _import_archive(self, source: Path) -> ImportResult:
|
||||||
|
# Layer-3 isolation: bomb-size guard + integrity test in a
|
||||||
|
# spawned child BEFORE extracting in this process. A
|
||||||
|
# decompression bomb or a native-lib crash on a malformed
|
||||||
|
# archive is contained to the child; we reject the file cleanly
|
||||||
|
# instead of OOMing/segfaulting the import worker. extract_archive
|
||||||
|
# is already fail-soft for plain exceptions, so this only adds
|
||||||
|
# the hard-crash protection.
|
||||||
|
probe = safe_probe.probe_archive(source)
|
||||||
|
if not probe.ok:
|
||||||
|
if probe.crashed:
|
||||||
|
return ImportResult(
|
||||||
|
status="failed",
|
||||||
|
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||||
|
)
|
||||||
|
# Clean rejection (bomb cap exceeded, integrity mismatch):
|
||||||
|
# still preserve the archive file itself as an attachment so
|
||||||
|
# nothing silently vanishes, matching extract_archive's
|
||||||
|
# fail-soft contract.
|
||||||
|
artist = self._resolve_artist(source)
|
||||||
|
post = self._post_for_sidecar(source, artist)
|
||||||
|
self._capture_attachment(source, post=post, artist=artist, resolved=True)
|
||||||
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
artist = self._resolve_artist(source)
|
artist = self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist)
|
post = self._post_for_sidecar(source, artist)
|
||||||
member_ids: list[int] = []
|
member_ids: list[int] = []
|
||||||
@@ -446,7 +470,25 @@ class Importer:
|
|||||||
# Compute file dimensions (images only) and apply filters.
|
# Compute file dimensions (images only) and apply filters.
|
||||||
width = height = None
|
width = height = None
|
||||||
has_alpha = False
|
has_alpha = False
|
||||||
if not is_video(source):
|
if is_video(source):
|
||||||
|
# Layer-3 isolation: validate the container via ffprobe (a
|
||||||
|
# separate process) before the rest of the pipeline touches
|
||||||
|
# it. A corrupt video that would crash a decoder is rejected
|
||||||
|
# cleanly here, and we capture width/height for free (the
|
||||||
|
# importer didn't previously record video dimensions).
|
||||||
|
probe = safe_probe.probe_video(source)
|
||||||
|
if not probe.ok:
|
||||||
|
if probe.crashed:
|
||||||
|
return ImportResult(
|
||||||
|
status="failed",
|
||||||
|
error=f"video probe crashed/timed out: {probe.reason}",
|
||||||
|
)
|
||||||
|
return ImportResult(
|
||||||
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||||
|
error=probe.reason,
|
||||||
|
)
|
||||||
|
width, height = probe.width, probe.height
|
||||||
|
else:
|
||||||
try:
|
try:
|
||||||
with Image.open(source) as im:
|
with Image.open(source) as im:
|
||||||
im.verify()
|
im.verify()
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Layer-2 one-shot re-download remediation for corrupt imported files.
|
||||||
|
|
||||||
|
When an import fails on a file that came from a known, pollable
|
||||||
|
subscription Source, deleting the bad copy and re-running the source's
|
||||||
|
downloader can fetch a fresh, unblemished copy. This only helps when:
|
||||||
|
|
||||||
|
- the corruption is in transit / on disk (not at the source), AND
|
||||||
|
- the file resolves to an ENABLED Source with a real feed URL
|
||||||
|
(a `sidecar:<platform>:<slug>` synthetic anchor is not pollable),
|
||||||
|
AND
|
||||||
|
- we haven't already re-fetched this task once (bounded by
|
||||||
|
ImportTask.refetched so source-side corruption can't loop).
|
||||||
|
|
||||||
|
Filesystem-only imports with no resolvable Source return 'no_source' —
|
||||||
|
the operator's only remediation there is to replace the file on disk.
|
||||||
|
|
||||||
|
Operator-requested 2026-05-28 (Layer 2).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import Artist, ImportTask, Source
|
||||||
|
from ..utils.paths import derive_top_level_artist
|
||||||
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||||
|
from ..utils.slug import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_refetch_source(
|
||||||
|
session: Session, source_path: str, import_root: Path,
|
||||||
|
) -> Source | None:
|
||||||
|
"""Find an enabled, real-URL Source for the file's (artist, platform),
|
||||||
|
or None when nothing re-pollable resolves."""
|
||||||
|
path = Path(source_path)
|
||||||
|
sc = find_sidecar(path)
|
||||||
|
if sc is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = json.loads(sc.read_text("utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
sd = parse_sidecar(data)
|
||||||
|
if not sd.platform:
|
||||||
|
return None
|
||||||
|
artist_name = derive_top_level_artist(path, import_root)
|
||||||
|
if not artist_name:
|
||||||
|
return None
|
||||||
|
artist = session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slugify(artist_name))
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
return None
|
||||||
|
src = session.execute(
|
||||||
|
select(Source)
|
||||||
|
.where(
|
||||||
|
Source.artist_id == artist.id,
|
||||||
|
Source.platform == sd.platform,
|
||||||
|
Source.enabled.is_(True),
|
||||||
|
)
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
).scalars().first()
|
||||||
|
if src is None:
|
||||||
|
return None
|
||||||
|
if (src.url or "").startswith("sidecar:"):
|
||||||
|
return None # synthetic anchor — not a pollable feed
|
||||||
|
return src
|
||||||
|
|
||||||
|
|
||||||
|
def attempt_refetch(
|
||||||
|
session: Session, task: ImportTask, import_root: Path,
|
||||||
|
) -> dict:
|
||||||
|
"""Delete the corrupt file, mark the task refetched, and trigger ONE
|
||||||
|
source re-check. Idempotent/bounded: a task already refetched (or
|
||||||
|
with no resolvable Source) is a no-op. Commits."""
|
||||||
|
if task.refetched:
|
||||||
|
return {"status": "already_refetched"}
|
||||||
|
src = resolve_refetch_source(session, task.source_path, import_root)
|
||||||
|
if src is None:
|
||||||
|
return {"status": "no_source"}
|
||||||
|
|
||||||
|
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
||||||
|
# the source re-check instead of skipping the still-present corrupt
|
||||||
|
# file.
|
||||||
|
try:
|
||||||
|
Path(task.source_path).unlink(missing_ok=True)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning("refetch unlink failed for %s: %s", task.source_path, exc)
|
||||||
|
|
||||||
|
task.refetched = True
|
||||||
|
session.add(task)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
# Lazy import to avoid a tasks→services→tasks import cycle at module
|
||||||
|
# load. download_source.delay() is sync-safe in any context.
|
||||||
|
from ..tasks.download import download_source
|
||||||
|
|
||||||
|
download_source.delay(src.id)
|
||||||
|
return {"status": "refetch_queued", "source_id": src.id}
|
||||||
@@ -64,30 +64,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
def _run_import_task(import_task_id: int) -> dict:
|
||||||
name="backend.app.tasks.import_file.import_media_file",
|
"""Shared body for import_media_file + import_archive_file. The two
|
||||||
bind=True,
|
tasks differ ONLY in their Celery time limits (a single media file
|
||||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
is sub-second; an archive runs the full per-member pipeline inline
|
||||||
retry_backoff=5,
|
for every member and can take many minutes). Both flip the row to
|
||||||
retry_backoff_max=60,
|
'processing', dispatch to `_do_import`, and honor the
|
||||||
retry_jitter=True,
|
flip-to-terminal resilience contract.
|
||||||
max_retries=3,
|
|
||||||
soft_time_limit=300,
|
|
||||||
time_limit=360,
|
|
||||||
)
|
|
||||||
def import_media_file(self, import_task_id: int) -> dict:
|
|
||||||
"""Returns a dict so the eager-mode tests can assert without DB.
|
|
||||||
|
|
||||||
Decorator notes:
|
|
||||||
- autoretry_for: transient DB / filesystem errors retry with
|
|
||||||
exponential backoff (5s base, jitter, max 3 attempts). On final
|
|
||||||
give-up the task raises and acks_late=True (set globally on the
|
|
||||||
Celery app) does NOT redeliver — the recovery sweep catches the
|
|
||||||
row instead.
|
|
||||||
- soft_time_limit (300s) raises SoftTimeLimitExceeded in this
|
|
||||||
process so the task can mark its row failed before being killed.
|
|
||||||
- time_limit (360s) is the hard cap; SIGKILL if the soft signal
|
|
||||||
was swallowed.
|
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
@@ -103,19 +86,85 @@ def import_media_file(self, import_task_id: int) -> dict:
|
|||||||
try:
|
try:
|
||||||
return _do_import(session, task, import_task_id)
|
return _do_import(session, task, import_task_id)
|
||||||
except SoftTimeLimitExceeded:
|
except SoftTimeLimitExceeded:
|
||||||
_mark_failed(session, task, "soft_time_limit exceeded (>300s)")
|
_mark_failed(session, task, "soft_time_limit exceeded")
|
||||||
raise
|
raise
|
||||||
except (OperationalError, DBAPIError, OSError):
|
except (OperationalError, DBAPIError, OSError):
|
||||||
# Retryable per the decorator; do NOT mark failed (let
|
# Retryable per the decorator; do NOT mark failed (let
|
||||||
# autoretry have a clean go at it). If autoretry exhausts,
|
# autoretry have a clean go at it). If autoretry exhausts,
|
||||||
# the row stays 'processing' and the maintenance sweep
|
# the row stays 'processing' and the maintenance sweep
|
||||||
# flips it within 5 min.
|
# flips it.
|
||||||
raise
|
raise
|
||||||
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
||||||
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.import_file.import_media_file",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
soft_time_limit=300,
|
||||||
|
time_limit=360,
|
||||||
|
)
|
||||||
|
def import_media_file(self, import_task_id: int) -> dict:
|
||||||
|
"""Import ONE media file (or non-media → PostAttachment). Sub-second
|
||||||
|
for the common case; the tight 5-min soft limit keeps a genuinely
|
||||||
|
stuck single-file import detectable fast.
|
||||||
|
|
||||||
|
Decorator notes:
|
||||||
|
- autoretry_for: transient DB / filesystem errors retry with
|
||||||
|
exponential backoff (5s base, jitter, max 3 attempts). On final
|
||||||
|
give-up the task raises and acks_late=True (set globally on the
|
||||||
|
Celery app) does NOT redeliver — the recovery sweep catches the
|
||||||
|
row instead.
|
||||||
|
- soft_time_limit (300s) raises SoftTimeLimitExceeded in-process
|
||||||
|
so the task can mark its row failed before being killed.
|
||||||
|
- time_limit (360s) is the hard SIGKILL cap.
|
||||||
|
"""
|
||||||
|
return _run_import_task(import_task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.import_file.import_archive_file",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
# Archives run the full per-member pipeline (sha256 + pHash + dedup
|
||||||
|
# query + copy + provenance) for EVERY media member inline, under a
|
||||||
|
# single task budget. A multi-hundred-member archive blows the
|
||||||
|
# 5-min media limit. soft=30min / hard=35min sizes for a large
|
||||||
|
# archive. Operator-flagged 2026-05-28 (target 1645019 hit the old
|
||||||
|
# shared 300s soft limit). The recovery sweep gives this task its
|
||||||
|
# own 40-min threshold via maintenance.TASK_STUCK_THRESHOLD_MINUTES
|
||||||
|
# so it isn't preempted while legitimately grinding through members.
|
||||||
|
soft_time_limit=1800,
|
||||||
|
time_limit=2100,
|
||||||
|
)
|
||||||
|
def import_archive_file(self, import_task_id: int) -> dict:
|
||||||
|
"""Import an archive: extract + run the per-member media pipeline for
|
||||||
|
every member inline, then preserve the archive as a PostAttachment.
|
||||||
|
Same body as import_media_file (dispatch is by file kind inside
|
||||||
|
Importer.import_one); split out purely for the larger time budget."""
|
||||||
|
return _run_import_task(import_task_id)
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_import(task_id: int, task_type: str) -> None:
|
||||||
|
"""Route an ImportTask to the right Celery task by its task_type.
|
||||||
|
Single source of truth for the media-vs-archive dispatch so the
|
||||||
|
scan, retry, and recovery-requeue paths stay in sync."""
|
||||||
|
if task_type == "archive":
|
||||||
|
import_archive_file.delay(task_id)
|
||||||
|
else:
|
||||||
|
import_media_file.delay(task_id)
|
||||||
|
|
||||||
|
|
||||||
def _do_import(session, task, import_task_id: int) -> dict:
|
def _do_import(session, task, import_task_id: int) -> dict:
|
||||||
"""Actual work, called from inside the resilience wrapper."""
|
"""Actual work, called from inside the resilience wrapper."""
|
||||||
settings = session.execute(
|
settings = session.execute(
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import delete, select, update
|
from sqlalchemy import and_, delete, or_, select, update
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
||||||
@@ -16,6 +17,22 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
|||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
STUCK_THRESHOLD_MINUTES = 5
|
STUCK_THRESHOLD_MINUTES = 5
|
||||||
|
# Archive ImportTasks run the per-member pipeline inline for every
|
||||||
|
# member (import_archive_file: soft=30min/hard=35min). The ImportTask
|
||||||
|
# 'processing' recovery sweep must give them a longer threshold or it
|
||||||
|
# re-queues a legitimately-running archive mid-import (double-process).
|
||||||
|
# 40 min = 5-min buffer past the archive task's hard kill.
|
||||||
|
# Operator-flagged 2026-05-28 (target 1645019, a big archive).
|
||||||
|
ARCHIVE_STUCK_THRESHOLD_MINUTES = 40
|
||||||
|
|
||||||
|
# Poison-pill cap. After being recovered (re-queued from a stuck
|
||||||
|
# 'processing' state) MAX_RECOVERY_ATTEMPTS-1 times, the next sweep
|
||||||
|
# marks the row 'failed' instead of looping. 3 = two recoveries then
|
||||||
|
# give up. A row reaches this only if it leaves NO terminal flip each
|
||||||
|
# run — i.e. it hard-crashes the worker (OOM/segfault/SIGKILL), the
|
||||||
|
# signature of a corrupt or oversized input. Caught exceptions already
|
||||||
|
# flip to terminal 'failed' and never enter this loop.
|
||||||
|
MAX_RECOVERY_ATTEMPTS = 3
|
||||||
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
||||||
OLD_TASK_DAYS = 7
|
OLD_TASK_DAYS = 7
|
||||||
PHASH_PAGE = 500
|
PHASH_PAGE = 500
|
||||||
@@ -24,17 +41,40 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
|||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
|
||||||
|
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
||||||
|
# Tasks/queues that legitimately run longer than the default 5-min
|
||||||
|
# threshold need their own larger value, else the sweep marks in-flight
|
||||||
|
# work 'error' before it finishes. Each value MUST be ≥ the relevant
|
||||||
|
# task.time_limit + a small buffer. task_name overrides take precedence
|
||||||
|
# over queue overrides.
|
||||||
|
#
|
||||||
|
# ml queue: tag_and_embed video branch (≈20 GPU ops); time_limit=1200.
|
||||||
|
# import_archive_file: shares the 'import' queue with the fast
|
||||||
|
# single-file import_media_file, so it needs a task-name override
|
||||||
|
# (the import queue itself stays at the 5-min default for single
|
||||||
|
# files); time_limit=2100.
|
||||||
|
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
|
"ml": 25,
|
||||||
|
}
|
||||||
|
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
|
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||||
def recover_interrupted_tasks() -> int:
|
def recover_interrupted_tasks() -> int:
|
||||||
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||||
|
|
||||||
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
|
1. 'processing' too long — worker crash mid-import. Re-queue via
|
||||||
.delay() and let the import retry. Was 30 min historically;
|
enqueue_import (routing media vs archive) and let the import
|
||||||
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
|
retry. Threshold is task-type-aware: media files are sub-second
|
||||||
import_media_file is sub-second for the vast majority of files and
|
and capped at the 5-min soft limit, so STUCK_THRESHOLD_MINUTES
|
||||||
capped at the per-task soft_time_limit (5 min), so anything still
|
(5) means a confirmed crash; archives run the per-member
|
||||||
'processing' after that window is a confirmed crash.
|
pipeline inline (import_archive_file, 35-min hard limit) so they
|
||||||
|
get ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid re-queueing a
|
||||||
|
still-running archive. (Media was tightened from 30 min to 5
|
||||||
|
2026-05-24 after a 2224-row zombie pile; archive split out
|
||||||
|
2026-05-28.)
|
||||||
|
|
||||||
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||||
creates rows with status='pending' (commit), then in a second pass
|
creates rows with status='pending' (commit), then in a second pass
|
||||||
@@ -51,7 +91,8 @@ def recover_interrupted_tasks() -> int:
|
|||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
media_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
archive_cutoff = now - timedelta(minutes=ARCHIVE_STUCK_THRESHOLD_MINUTES)
|
||||||
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
||||||
@@ -59,20 +100,67 @@ def recover_interrupted_tasks() -> int:
|
|||||||
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
||||||
# /import deep scan piled up orphans). Folding the SELECT into the
|
# /import deep scan piled up orphans). Folding the SELECT into the
|
||||||
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
||||||
# exactly the ids that flipped so the stuck sweep can still
|
# exactly the (id, task_type) pairs that flipped so the requeue
|
||||||
# .delay() each one.
|
# can route media vs archive correctly.
|
||||||
stuck_result = session.execute(
|
#
|
||||||
|
# Media + archive get separate cutoffs: a single media file is
|
||||||
|
# sub-second so 5 min means crash; an archive runs the per-member
|
||||||
|
# pipeline inline and can legitimately take up to its 35-min hard
|
||||||
|
# limit, so it gets ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid
|
||||||
|
# re-queueing a still-running archive.
|
||||||
|
stuck_predicate = and_(
|
||||||
|
ImportTask.status == "processing",
|
||||||
|
or_(
|
||||||
|
and_(ImportTask.task_type != "archive",
|
||||||
|
ImportTask.started_at < media_cutoff),
|
||||||
|
and_(ImportTask.task_type == "archive",
|
||||||
|
ImportTask.started_at < archive_cutoff),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# POISON-PILL CIRCUIT BREAKER (Layer 1, 2026-05-28). A row that
|
||||||
|
# leaves no terminal flip (hard worker crash: OOM/segfault/SIGKILL
|
||||||
|
# on a corrupt or oversized input) gets re-queued by this sweep —
|
||||||
|
# and would loop forever, re-crashing the worker each pass,
|
||||||
|
# without a cap. Once a row has already been recovered
|
||||||
|
# MAX_RECOVERY_ATTEMPTS-1 times, stop re-queueing it and mark it
|
||||||
|
# 'failed' with a diagnostic so the operator can find + replace
|
||||||
|
# the offending file. This UPDATE runs FIRST so the rows it
|
||||||
|
# claims drop out of 'processing' before the re-queue pass.
|
||||||
|
poison_result = session.execute(
|
||||||
update(ImportTask)
|
update(ImportTask)
|
||||||
.where(ImportTask.status == "processing")
|
.where(stuck_predicate)
|
||||||
.where(ImportTask.started_at < processing_cutoff)
|
.where(ImportTask.recovery_count >= MAX_RECOVERY_ATTEMPTS - 1)
|
||||||
.values(
|
.values(
|
||||||
status="queued",
|
status="failed",
|
||||||
started_at=None,
|
finished_at=now,
|
||||||
error="recovered from stuck state",
|
error=(
|
||||||
|
f"crashed or stalled the worker {MAX_RECOVERY_ATTEMPTS} "
|
||||||
|
f"times without completing — likely a corrupt or "
|
||||||
|
f"oversized input. Not re-queued. Inspect/replace the "
|
||||||
|
f"file, then retry via /api/import/retry-failed."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.returning(ImportTask.id)
|
.returning(ImportTask.id)
|
||||||
)
|
)
|
||||||
stuck_ids = [row[0] for row in stuck_result.all()]
|
poison_ids = [r[0] for r in poison_result.all()]
|
||||||
|
|
||||||
|
# Re-queue the remaining stuck rows (under the cap) and bump
|
||||||
|
# their recovery_count. RETURNING (id, task_type) so the requeue
|
||||||
|
# routes media vs archive correctly.
|
||||||
|
stuck_result = session.execute(
|
||||||
|
update(ImportTask)
|
||||||
|
.where(stuck_predicate)
|
||||||
|
.where(ImportTask.recovery_count < MAX_RECOVERY_ATTEMPTS - 1)
|
||||||
|
.values(
|
||||||
|
status="queued",
|
||||||
|
started_at=None,
|
||||||
|
recovery_count=ImportTask.recovery_count + 1,
|
||||||
|
error="recovered from stuck state",
|
||||||
|
)
|
||||||
|
.returning(ImportTask.id, ImportTask.task_type)
|
||||||
|
)
|
||||||
|
stuck = stuck_result.all()
|
||||||
|
|
||||||
orphan_result = session.execute(
|
orphan_result = session.execute(
|
||||||
update(ImportTask)
|
update(ImportTask)
|
||||||
@@ -91,12 +179,34 @@ def recover_interrupted_tasks() -> int:
|
|||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
if stuck_ids:
|
if stuck:
|
||||||
from .import_file import import_media_file
|
from .import_file import enqueue_import
|
||||||
for tid in stuck_ids:
|
for tid, task_type in stuck:
|
||||||
import_media_file.delay(tid)
|
enqueue_import(tid, task_type)
|
||||||
|
|
||||||
return len(stuck_ids) + orphan_count
|
# Layer-2 auto re-download (env-gated, default OFF). For each
|
||||||
|
# poison-pill row that resolves to a pollable Source, delete the
|
||||||
|
# bad file and trigger ONE source re-check to fetch a fresh
|
||||||
|
# copy. Bounded by ImportTask.refetched so source-side
|
||||||
|
# corruption can't loop. The 'failed' row stays as history; the
|
||||||
|
# re-downloaded file re-imports as a fresh task on the next scan.
|
||||||
|
if poison_ids and os.environ.get("FC_AUTO_REFETCH_CORRUPT", "0") == "1":
|
||||||
|
from ..models import ImportSettings
|
||||||
|
from ..services.refetch_service import attempt_refetch
|
||||||
|
import_root = Path(session.execute(
|
||||||
|
select(ImportSettings.import_scan_path)
|
||||||
|
.where(ImportSettings.id == 1)
|
||||||
|
).scalar_one())
|
||||||
|
for pid in poison_ids:
|
||||||
|
ptask = session.get(ImportTask, pid)
|
||||||
|
if ptask is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
attempt_refetch(session, ptask, import_root)
|
||||||
|
except Exception as exc: # noqa: BLE001 — best-effort
|
||||||
|
log.warning("auto-refetch failed for task %s: %s", pid, exc)
|
||||||
|
|
||||||
|
return len(stuck) + len(poison_ids) + orphan_count
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||||
@@ -121,18 +231,29 @@ def cleanup_old_tasks() -> int:
|
|||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
def recover_stalled_task_runs() -> int:
|
def recover_stalled_task_runs() -> int:
|
||||||
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||||
to 'error'. FC-3i.
|
threshold to 'error'. FC-3i.
|
||||||
|
|
||||||
A row gets stuck when the worker dies without emitting
|
A row gets stuck when the worker dies without emitting
|
||||||
task_postrun / task_failure (e.g. OOM, container restart between
|
task_postrun / task_failure (e.g. OOM, container restart between
|
||||||
signals, signal handler raised+logged). Shares the 5-min threshold
|
signals, signal handler raised+logged). The default 5-min threshold
|
||||||
with recover_interrupted_tasks for consistency.
|
fits short-lived queues (import/thumbnail/download); queues that
|
||||||
|
legitimately run longer tasks (ml-video, deep scans) get their
|
||||||
|
own larger threshold via QUEUE_STUCK_THRESHOLD_MINUTES so the
|
||||||
|
sweep doesn't preempt them.
|
||||||
|
|
||||||
|
Runs once per distinct threshold value: each pass updates rows
|
||||||
|
whose queue maps to that threshold.
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
now = datetime.now(UTC)
|
||||||
with SessionLocal() as session:
|
override_tasks = set(TASK_STUCK_THRESHOLD_MINUTES.keys())
|
||||||
result = session.execute(
|
override_queues = set(QUEUE_STUCK_THRESHOLD_MINUTES.keys())
|
||||||
|
total = 0
|
||||||
|
|
||||||
|
def _flag(minutes, *extra_where):
|
||||||
|
cutoff = now - timedelta(minutes=minutes)
|
||||||
|
stmt = (
|
||||||
update(TaskRun)
|
update(TaskRun)
|
||||||
.where(TaskRun.status == "running")
|
.where(TaskRun.status == "running")
|
||||||
.where(TaskRun.started_at < cutoff)
|
.where(TaskRun.started_at < cutoff)
|
||||||
@@ -140,14 +261,42 @@ def recover_stalled_task_runs() -> int:
|
|||||||
status="error",
|
status="error",
|
||||||
error_type="RecoverySweep",
|
error_type="RecoverySweep",
|
||||||
error_message=(
|
error_message=(
|
||||||
f"no completion signal received within "
|
f"no completion signal received within {minutes} min"
|
||||||
f"{STUCK_THRESHOLD_MINUTES} min"
|
|
||||||
),
|
),
|
||||||
finished_at=datetime.now(UTC),
|
finished_at=now,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
for w in extra_where:
|
||||||
|
stmt = stmt.where(w)
|
||||||
|
return session.execute(stmt).rowcount or 0
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
# Precedence: task_name override → queue override → default.
|
||||||
|
# Each pass excludes rows claimed by a higher-precedence pass so
|
||||||
|
# every row is touched at most once.
|
||||||
|
|
||||||
|
# 1. Per-task-name overrides (e.g. import_archive_file, which
|
||||||
|
# shares the 'import' queue with fast single-file imports).
|
||||||
|
for task_name, minutes in TASK_STUCK_THRESHOLD_MINUTES.items():
|
||||||
|
total += _flag(minutes, TaskRun.task_name == task_name)
|
||||||
|
|
||||||
|
# 2. Per-queue overrides, excluding the override task-names.
|
||||||
|
for queue, minutes in QUEUE_STUCK_THRESHOLD_MINUTES.items():
|
||||||
|
wheres = [TaskRun.queue == queue]
|
||||||
|
if override_tasks:
|
||||||
|
wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||||
|
total += _flag(minutes, *wheres)
|
||||||
|
|
||||||
|
# 3. Default — everything not claimed above.
|
||||||
|
default_wheres = []
|
||||||
|
if override_queues:
|
||||||
|
default_wheres.append(TaskRun.queue.notin_(override_queues))
|
||||||
|
if override_tasks:
|
||||||
|
default_wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||||
|
total += _flag(STUCK_THRESHOLD_MINUTES, *default_wheres)
|
||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
return result.rowcount or 0
|
return total
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||||
|
|||||||
+21
-2
@@ -31,8 +31,15 @@ def _is_video(path: Path) -> bool:
|
|||||||
retry_backoff_max=60,
|
retry_backoff_max=60,
|
||||||
retry_jitter=True,
|
retry_jitter=True,
|
||||||
max_retries=3,
|
max_retries=3,
|
||||||
soft_time_limit=300,
|
# Sized for the video branch: sample 10 frames, run tagger +
|
||||||
time_limit=420,
|
# embedder on each (≈20 GPU ops vs 2 for an image). A loaded
|
||||||
|
# ml-worker can take 5-10 min on a long video; bumped from
|
||||||
|
# 5min/7min on 2026-05-28 after operator-flagged image 6288 (a
|
||||||
|
# .mp4) hit the recovery sweep at 5 min while still legitimately
|
||||||
|
# processing. Image runs return in seconds; the bump doesn't
|
||||||
|
# affect their UX.
|
||||||
|
soft_time_limit=900, # 15 min
|
||||||
|
time_limit=1200, # 20 min hard
|
||||||
)
|
)
|
||||||
def tag_and_embed(self, image_id: int) -> dict:
|
def tag_and_embed(self, image_id: int) -> dict:
|
||||||
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
||||||
@@ -64,6 +71,18 @@ def tag_and_embed(self, image_id: int) -> dict:
|
|||||||
embedder = get_embedder()
|
embedder = get_embedder()
|
||||||
|
|
||||||
if _is_video(src):
|
if _is_video(src):
|
||||||
|
# Layer-3 isolation: ffprobe (a separate process) validates
|
||||||
|
# the container before we burn ~20 GPU ops sampling frames
|
||||||
|
# from it. A corrupt video that would crash the frame
|
||||||
|
# decoder is rejected cleanly here instead of taking down
|
||||||
|
# the ml-worker. Operator-flagged 2026-05-28.
|
||||||
|
from ..utils import safe_probe
|
||||||
|
vprobe = safe_probe.probe_video(src)
|
||||||
|
if not vprobe.ok:
|
||||||
|
return {
|
||||||
|
"status": "bad_video", "image_id": image_id,
|
||||||
|
"reason": vprobe.reason,
|
||||||
|
}
|
||||||
frames = _sample_video_frames(
|
frames = _sample_video_frames(
|
||||||
src, int(os.environ.get("VIDEO_ML_FRAMES", "10"))
|
src, int(os.environ.get("VIDEO_ML_FRAMES", "10"))
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
|
|||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..config import get_config
|
from ..config import get_config
|
||||||
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
||||||
|
from ..services.archive_extractor import is_archive
|
||||||
from ..services.scheduler_service import select_due_sources
|
from ..services.scheduler_service import select_due_sources
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
@@ -96,7 +97,9 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
task = ImportTask(
|
task = ImportTask(
|
||||||
batch_id=batch_id,
|
batch_id=batch_id,
|
||||||
source_path=entry_str,
|
source_path=entry_str,
|
||||||
task_type="media",
|
# Archives route to import_archive_file (larger time
|
||||||
|
# budget) — they run the per-member pipeline inline.
|
||||||
|
task_type="archive" if is_archive(entry) else "media",
|
||||||
status="pending",
|
status="pending",
|
||||||
size_bytes=size,
|
size_bytes=size,
|
||||||
)
|
)
|
||||||
@@ -115,15 +118,16 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
batch.finished_at = datetime.now(UTC)
|
batch.finished_at = datetime.now(UTC)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
# Now enqueue import_media_file for each pending task.
|
# Now enqueue each pending task on the right Celery task
|
||||||
|
# (media vs archive) via the shared router.
|
||||||
|
from .import_file import enqueue_import
|
||||||
|
|
||||||
for task in session.execute(
|
for task in session.execute(
|
||||||
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
||||||
).scalars():
|
).scalars():
|
||||||
task.status = "queued"
|
task.status = "queued"
|
||||||
session.add(task)
|
session.add(task)
|
||||||
from .import_file import import_media_file
|
enqueue_import(task.id, task.task_type)
|
||||||
|
|
||||||
import_media_file.delay(task.id)
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
if mode == "deep":
|
if mode == "deep":
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Subprocess-isolated media probes (Layer 3 of import resilience).
|
||||||
|
|
||||||
|
A malformed video or archive can hard-crash the worker process — a
|
||||||
|
decoder OOM, a native-lib segfault, or a decompression bomb. A hard
|
||||||
|
crash leaves no terminal flip, so the recovery sweep re-queues the row
|
||||||
|
and it crashes again: a poison-pill loop (the Layer-1 cap is the
|
||||||
|
backstop, but isolating the crash is better — the file gets a clean
|
||||||
|
terminal failure and the worker never dies).
|
||||||
|
|
||||||
|
These probes run the risky read in a way that contains the blast:
|
||||||
|
|
||||||
|
- Video: `ffprobe` is a separate binary, so a crash decoding the
|
||||||
|
container kills only ffprobe (non-zero exit), never the worker. Also
|
||||||
|
returns width/height, which the importer didn't previously capture
|
||||||
|
for videos.
|
||||||
|
- Archive: an uncompressed-size guard (catches decompression bombs
|
||||||
|
before they OOM anything) plus an integrity test in a spawned child
|
||||||
|
(catches native-lib crashes on a malformed archive). A child segfault
|
||||||
|
/ OOM shows up as a non-zero exit code, not a dead worker.
|
||||||
|
|
||||||
|
Images are intentionally NOT probed here: Pillow raises (it doesn't
|
||||||
|
segfault) on the realistic corrupt-image cases, the importer already
|
||||||
|
catches that as an invalid_image skip, and a subprocess per image would
|
||||||
|
wreck deep-scan throughput on a large library. Add an image branch only
|
||||||
|
if a real image-induced worker crash is ever observed.
|
||||||
|
|
||||||
|
Operator-requested 2026-05-28 (Layer 3).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import multiprocessing as mp
|
||||||
|
import subprocess
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
VIDEO_PROBE_TIMEOUT_SECONDS = 60
|
||||||
|
ARCHIVE_PROBE_TIMEOUT_SECONDS = 120
|
||||||
|
# Refuse archives whose total UNCOMPRESSED size exceeds this — the
|
||||||
|
# classic decompression-bomb guard (a 4 GB cap comfortably clears real
|
||||||
|
# art-pack archives while stopping a few-KB zip that expands to TB).
|
||||||
|
MAX_ARCHIVE_UNCOMPRESSED_BYTES = 4 * 1024 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProbeResult:
|
||||||
|
ok: bool
|
||||||
|
# crashed=True means the probe HARD-FAILED (subprocess killed by a
|
||||||
|
# signal, OOM, or timeout) — the poison-pill signature. crashed=False
|
||||||
|
# with ok=False means a clean rejection (corrupt-but-handled,
|
||||||
|
# bomb-size-exceeded, integrity mismatch). Callers map crashed → a
|
||||||
|
# terminal 'failed', clean → a 'skipped'/'failed' of their choosing.
|
||||||
|
crashed: bool = False
|
||||||
|
reason: str | None = None
|
||||||
|
width: int | None = None
|
||||||
|
height: int | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||||
|
"""Validate a video container + first video stream via ffprobe."""
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
[
|
||||||
|
"ffprobe", "-v", "error",
|
||||||
|
"-select_streams", "v:0",
|
||||||
|
"-show_entries", "stream=width,height",
|
||||||
|
"-of", "json", str(path),
|
||||||
|
],
|
||||||
|
capture_output=True, text=True, timeout=timeout,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return ProbeResult(ok=False, crashed=True, reason="ffprobe timed out")
|
||||||
|
except OSError as exc:
|
||||||
|
# ffprobe missing / not executable — environmental, not the
|
||||||
|
# file's fault. Treat as a clean non-crash failure so the import
|
||||||
|
# path can decide (it currently proceeds without dims).
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe unavailable: {exc}")
|
||||||
|
if out.returncode != 0:
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=False,
|
||||||
|
reason=f"ffprobe rejected the file: {out.stderr.strip()[:200]}",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
streams = (json.loads(out.stdout) or {}).get("streams") or []
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe output parse failed: {exc}")
|
||||||
|
if not streams:
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason="no decodable video stream")
|
||||||
|
return ProbeResult(
|
||||||
|
ok=True, width=streams[0].get("width"), height=streams[0].get("height"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def probe_archive(path: Path, *, timeout: float = ARCHIVE_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||||
|
"""Bomb-size guard + isolated integrity test for an archive."""
|
||||||
|
ctx = mp.get_context("spawn")
|
||||||
|
q = ctx.Queue()
|
||||||
|
proc = ctx.Process(target=_archive_probe_target, args=(str(path), q))
|
||||||
|
proc.start()
|
||||||
|
proc.join(timeout)
|
||||||
|
if proc.is_alive():
|
||||||
|
proc.terminate()
|
||||||
|
proc.join(5)
|
||||||
|
return ProbeResult(ok=False, crashed=True, reason="archive probe timed out")
|
||||||
|
if proc.exitcode != 0:
|
||||||
|
# Negative exitcode = killed by signal (segfault); positive =
|
||||||
|
# the child os._exit'd or was OOM-killed. Either way the file
|
||||||
|
# hard-crashed the probe — the poison-pill signature.
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=True,
|
||||||
|
reason=f"archive probe crashed (exit {proc.exitcode})",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
outcome = q.get(timeout=5)
|
||||||
|
except Exception: # noqa: BLE001 — empty queue / broken pipe
|
||||||
|
return ProbeResult(ok=False, crashed=True, reason="archive probe produced no result")
|
||||||
|
status, detail = outcome
|
||||||
|
if status == "ok":
|
||||||
|
return ProbeResult(ok=True)
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason=detail)
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_probe_target(path_str: str, q) -> None:
|
||||||
|
"""Runs in the spawned child. Reads member sizes (bomb guard) then
|
||||||
|
runs the format's integrity test. Puts ('ok', None) or
|
||||||
|
('error', reason). A crash/OOM here never reaches the queue — the
|
||||||
|
parent reads the non-zero exit code instead."""
|
||||||
|
path = Path(path_str)
|
||||||
|
ext = path.suffix.lower()
|
||||||
|
try:
|
||||||
|
total, test_bad = _inspect_archive(path, ext)
|
||||||
|
except Exception as exc: # noqa: BLE001 — clean rejection
|
||||||
|
q.put(("error", f"{type(exc).__name__}: {exc}"))
|
||||||
|
return
|
||||||
|
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||||
|
gib = total / (1024 ** 3)
|
||||||
|
q.put(("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap"))
|
||||||
|
return
|
||||||
|
if test_bad is not None:
|
||||||
|
q.put(("error", f"integrity test failed at member {test_bad!r}"))
|
||||||
|
return
|
||||||
|
q.put(("ok", None))
|
||||||
|
|
||||||
|
|
||||||
|
def _inspect_archive(path: Path, ext: str):
|
||||||
|
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||||
|
for the archive. Format-specific; raises on a structurally-broken
|
||||||
|
container (caught by the child as a clean rejection)."""
|
||||||
|
if ext in (".zip", ".cbz"):
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
with zipfile.ZipFile(path) as zf:
|
||||||
|
total = sum(zi.file_size for zi in zf.infolist())
|
||||||
|
return total, zf.testzip()
|
||||||
|
if ext == ".rar":
|
||||||
|
import rarfile
|
||||||
|
|
||||||
|
with rarfile.RarFile(path) as rf:
|
||||||
|
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||||
|
rf.testrar()
|
||||||
|
return total, None
|
||||||
|
if ext == ".7z":
|
||||||
|
import py7zr
|
||||||
|
|
||||||
|
with py7zr.SevenZipFile(path, "r") as zf:
|
||||||
|
info = zf.archiveinfo()
|
||||||
|
total = getattr(info, "uncompressed", None)
|
||||||
|
ok = zf.test() # True / None when all members pass
|
||||||
|
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||||
|
# Unknown extension — nothing to test; treat as clean.
|
||||||
|
return None, None
|
||||||
@@ -1,47 +1,118 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-dl-row" @click="$emit('open', event.id)">
|
<div
|
||||||
<v-icon :icon="statusIcon" :color="statusColor" size="small" />
|
class="fc-dl-row"
|
||||||
|
:class="[`fc-dl-row--${event.status || 'unknown'}`]"
|
||||||
|
@click="$emit('open', event.id)"
|
||||||
|
>
|
||||||
|
<!-- Colored left edge marks the run's status; matches the row's
|
||||||
|
status-chip color but reads at a glance without needing to
|
||||||
|
parse the chip text. -->
|
||||||
|
<div class="fc-dl-row__bar" />
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
:color="statusColor"
|
||||||
|
size="small"
|
||||||
|
variant="tonal"
|
||||||
|
:prepend-icon="statusIcon"
|
||||||
|
class="fc-dl-row__status"
|
||||||
|
>{{ statusLabel }}</v-chip>
|
||||||
|
|
||||||
<RouterLink
|
<RouterLink
|
||||||
v-if="event.artist_slug"
|
v-if="event.artist_slug"
|
||||||
:to="`/artist/${event.artist_slug}`"
|
:to="`/artist/${event.artist_slug}`"
|
||||||
class="fc-dl-row__artist"
|
class="fc-dl-row__artist"
|
||||||
@click.stop
|
@click.stop
|
||||||
>{{ event.artist_name }}</RouterLink>
|
>{{ event.artist_name }}</RouterLink>
|
||||||
<span v-else class="fc-dl-row__artist">—</span>
|
<span v-else class="fc-dl-row__artist fc-dl-row__artist--missing">—</span>
|
||||||
<v-chip size="x-small" variant="tonal">{{ event.platform || '—' }}</v-chip>
|
|
||||||
<span class="fc-dl-row__time">{{ fmtTime(event.started_at) }}</span>
|
<PlatformChip
|
||||||
<span class="fc-dl-row__files">{{ event.files_count }} files</span>
|
v-if="event.platform"
|
||||||
<span class="fc-dl-row__duration">{{ fmtDuration(event.summary?.duration_seconds) }}</span>
|
:platform="event.platform"
|
||||||
<span v-if="event.error" class="fc-dl-row__error">{{ event.error }}</span>
|
size="x-small"
|
||||||
|
class="fc-dl-row__platform"
|
||||||
|
/>
|
||||||
|
<span v-else class="fc-dl-row__platform-missing">—</span>
|
||||||
|
|
||||||
|
<span class="fc-dl-row__time" :title="event.started_at">
|
||||||
|
{{ fmtTime(event.started_at) }}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
v-if="event.files_count > 0"
|
||||||
|
size="x-small" variant="tonal" color="info"
|
||||||
|
prepend-icon="mdi-image-multiple"
|
||||||
|
class="fc-dl-row__files"
|
||||||
|
>{{ event.files_count }}</v-chip>
|
||||||
|
<span v-else class="fc-dl-row__no-files" aria-label="no new files">·</span>
|
||||||
|
|
||||||
|
<span class="fc-dl-row__duration">
|
||||||
|
{{ fmtDuration(event.summary?.duration_seconds) }}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
v-if="event.error"
|
||||||
|
color="error" size="x-small" variant="tonal"
|
||||||
|
prepend-icon="mdi-alert-octagon"
|
||||||
|
class="fc-dl-row__error"
|
||||||
|
:title="event.error"
|
||||||
|
>{{ truncateError(event.error) }}</v-chip>
|
||||||
|
<span v-else class="fc-dl-row__error-spacer" />
|
||||||
|
|
||||||
|
<div class="fc-dl-row__actions" @click.stop>
|
||||||
|
<v-btn
|
||||||
|
v-if="event.status === 'error' && event.source_id"
|
||||||
|
icon size="x-small" variant="text" color="warning"
|
||||||
|
:loading="retrying"
|
||||||
|
@click.stop="onRetry"
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-refresh</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Retry source check</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
<v-btn icon size="x-small" variant="text" @click.stop="$emit('open', event.id)">
|
||||||
|
<v-icon size="small">mdi-information-outline</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Details</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
v-if="event.artist_slug"
|
||||||
|
icon size="x-small" variant="text"
|
||||||
|
:to="`/artist/${event.artist_slug}`"
|
||||||
|
@click.stop
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-account-circle</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Open artist</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import { RouterLink } from 'vue-router'
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import PlatformChip from '../subscriptions/PlatformChip.vue'
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
|
||||||
const props = defineProps({ event: { type: Object, required: true } })
|
const props = defineProps({ event: { type: Object, required: true } })
|
||||||
defineEmits(['open'])
|
defineEmits(['open'])
|
||||||
|
|
||||||
const statusIcon = computed(() => ({
|
const sourcesStore = useSourcesStore()
|
||||||
ok: 'mdi-check-circle',
|
const retrying = ref(false)
|
||||||
error: 'mdi-alert-circle',
|
|
||||||
running: 'mdi-progress-clock',
|
|
||||||
pending: 'mdi-clock-outline',
|
|
||||||
skipped: 'mdi-minus-circle',
|
|
||||||
}[props.event.status] || 'mdi-help-circle'))
|
|
||||||
|
|
||||||
const statusColor = computed(() => ({
|
const _STATUS = {
|
||||||
ok: 'success',
|
ok: { color: 'success', icon: 'mdi-check-circle', label: 'Completed' },
|
||||||
error: 'error',
|
error: { color: 'error', icon: 'mdi-alert-circle', label: 'Failed' },
|
||||||
running: 'info',
|
running: { color: 'info', icon: 'mdi-progress-clock', label: 'Running' },
|
||||||
pending: 'secondary',
|
pending: { color: 'grey', icon: 'mdi-clock-outline', label: 'Queued' },
|
||||||
skipped: 'warning',
|
skipped: { color: 'warning', icon: 'mdi-skip-next', label: 'Skipped' },
|
||||||
}[props.event.status] || undefined))
|
}
|
||||||
|
const statusColor = computed(() => _STATUS[props.event.status]?.color || 'grey')
|
||||||
|
const statusIcon = computed(() => _STATUS[props.event.status]?.icon || 'mdi-help-circle')
|
||||||
|
const statusLabel = computed(() => _STATUS[props.event.status]?.label || props.event.status)
|
||||||
|
|
||||||
function fmtTime(iso) {
|
function fmtTime(iso) {
|
||||||
if (!iso) return '—'
|
if (!iso) return '—'
|
||||||
return iso.slice(0, 19).replace('T', ' ')
|
// 2026-05-27 23:36 — second granularity is in the row's title attr
|
||||||
|
return iso.slice(0, 16).replace('T', ' ')
|
||||||
}
|
}
|
||||||
function fmtDuration(sec) {
|
function fmtDuration(sec) {
|
||||||
if (sec == null) return '—'
|
if (sec == null) return '—'
|
||||||
@@ -49,34 +120,107 @@ function fmtDuration(sec) {
|
|||||||
const m = Math.floor(sec / 60), s = Math.floor(sec % 60)
|
const m = Math.floor(sec / 60), s = Math.floor(sec % 60)
|
||||||
return `${m}m ${s}s`
|
return `${m}m ${s}s`
|
||||||
}
|
}
|
||||||
|
function truncateError(msg) {
|
||||||
|
const s = String(msg || '')
|
||||||
|
if (s.length <= 60) return s
|
||||||
|
return s.slice(0, 57) + '…'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onRetry() {
|
||||||
|
if (!props.event.source_id) return
|
||||||
|
retrying.value = true
|
||||||
|
try {
|
||||||
|
await sourcesStore.checkNow(props.event.source_id)
|
||||||
|
globalThis.window?.__fcToast?.({
|
||||||
|
text: `Source check re-queued`, type: 'success',
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
const isInFlight = !!e?.body?.download_event_id
|
||||||
|
globalThis.window?.__fcToast?.({
|
||||||
|
text: isInFlight ? 'Already running' : `Retry failed: ${e?.detail || e?.message || e}`,
|
||||||
|
type: isInFlight ? 'info' : 'error',
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
retrying.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-dl-row {
|
.fc-dl-row {
|
||||||
|
position: relative;
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: 24px 1fr 96px 160px 80px 80px 1fr;
|
grid-template-columns:
|
||||||
gap: 0.75rem;
|
/* bar */ 4px
|
||||||
|
/* status */ 120px
|
||||||
|
/* artist */ minmax(120px, 1.2fr)
|
||||||
|
/* plat */ 140px
|
||||||
|
/* time */ 140px
|
||||||
|
/* files */ 60px
|
||||||
|
/* dur */ 70px
|
||||||
|
/* error */ minmax(0, 1.5fr)
|
||||||
|
/* actions*/ 120px;
|
||||||
|
gap: 0.6rem;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
padding: 0.5rem 0.75rem;
|
padding: 0.55rem 0.75rem 0.55rem 0;
|
||||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.18);
|
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
|
transition: background 0.12s ease;
|
||||||
}
|
}
|
||||||
.fc-dl-row:hover { background: rgb(var(--v-theme-surface) / 0.5); }
|
.fc-dl-row:hover {
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||||
|
}
|
||||||
|
.fc-dl-row__bar {
|
||||||
|
width: 4px;
|
||||||
|
align-self: stretch;
|
||||||
|
border-radius: 0 2px 2px 0;
|
||||||
|
}
|
||||||
|
.fc-dl-row--ok .fc-dl-row__bar { background: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-dl-row--error .fc-dl-row__bar { background: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-dl-row--running .fc-dl-row__bar { background: rgb(var(--v-theme-info)); }
|
||||||
|
.fc-dl-row--skipped .fc-dl-row__bar { background: rgb(var(--v-theme-warning)); }
|
||||||
|
.fc-dl-row--pending .fc-dl-row__bar { background: rgb(var(--v-theme-on-surface-variant) / 0.4); }
|
||||||
|
|
||||||
|
.fc-dl-row__status { justify-self: start; }
|
||||||
.fc-dl-row__artist {
|
.fc-dl-row__artist {
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
|
font-weight: 500;
|
||||||
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-dl-row__artist--missing,
|
||||||
|
.fc-dl-row__platform-missing,
|
||||||
|
.fc-dl-row__no-files {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
opacity: 0.5;
|
||||||
}
|
}
|
||||||
.fc-dl-row__artist:hover { color: rgb(var(--v-theme-accent)); }
|
.fc-dl-row__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
.fc-dl-row__time, .fc-dl-row__files, .fc-dl-row__duration {
|
.fc-dl-row__platform { justify-self: start; }
|
||||||
|
.fc-dl-row__time,
|
||||||
|
.fc-dl-row__duration {
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
font-variant-numeric: tabular-nums;
|
font-variant-numeric: tabular-nums;
|
||||||
}
|
|
||||||
.fc-dl-row__error {
|
|
||||||
color: rgb(var(--v-theme-error));
|
|
||||||
font-size: 0.85rem;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
.fc-dl-row__no-files {
|
||||||
|
text-align: center;
|
||||||
|
font-size: 1.1rem;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error {
|
||||||
|
justify-self: start;
|
||||||
|
max-width: 100%;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error :deep(.v-chip__content) {
|
||||||
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error-spacer { /* keeps the grid column reserved */ }
|
||||||
|
|
||||||
|
.fc-dl-row__actions {
|
||||||
|
display: flex; gap: 2px;
|
||||||
|
justify-self: end;
|
||||||
|
opacity: 0.5;
|
||||||
|
transition: opacity 0.12s ease;
|
||||||
|
}
|
||||||
|
.fc-dl-row:hover .fc-dl-row__actions { opacity: 1; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -51,6 +51,19 @@
|
|||||||
title="Click for full error"
|
title="Click for full error"
|
||||||
>{{ shorten(item.error, 60) }}</button>
|
>{{ shorten(item.error, 60) }}</button>
|
||||||
</template>
|
</template>
|
||||||
|
<template #item.actions="{ item }">
|
||||||
|
<v-btn
|
||||||
|
v-if="item.status === 'failed'"
|
||||||
|
icon size="x-small" variant="text"
|
||||||
|
:loading="refetching === item.id"
|
||||||
|
@click="onRefetch(item)"
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-cloud-refresh</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">
|
||||||
|
Re-fetch original (re-download from source)
|
||||||
|
</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
</template>
|
||||||
</v-data-table-virtual>
|
</v-data-table-virtual>
|
||||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||||
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
||||||
@@ -149,9 +162,29 @@ const headers = [
|
|||||||
{ title: 'Source', key: 'source_path', sortable: false },
|
{ title: 'Source', key: 'source_path', sortable: false },
|
||||||
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
||||||
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
||||||
{ title: 'Note', key: 'error', sortable: false }
|
{ title: 'Note', key: 'error', sortable: false },
|
||||||
|
{ title: '', key: 'actions', sortable: false, width: 56 }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const refetching = ref(null)
|
||||||
|
const _REFETCH_MSG = {
|
||||||
|
refetch_queued: { text: 'Re-fetch queued — re-downloading from source', type: 'success' },
|
||||||
|
no_source: { text: 'No re-fetchable source (filesystem import — replace the file manually)', type: 'info' },
|
||||||
|
already_refetched: { text: 'Already re-fetched once', type: 'info' },
|
||||||
|
}
|
||||||
|
async function onRefetch(item) {
|
||||||
|
refetching.value = item.id
|
||||||
|
try {
|
||||||
|
const res = await store.refetchTask(item.id)
|
||||||
|
const msg = _REFETCH_MSG[res.status] || { text: `Re-fetch: ${res.status}`, type: 'info' }
|
||||||
|
window.__fcToast?.(msg)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Re-fetch failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
refetching.value = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||||
const hasStuck = computed(() => store.tasks.some(
|
const hasStuck = computed(() => store.tasks.some(
|
||||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||||
|
|||||||
@@ -20,15 +20,44 @@
|
|||||||
<v-progress-circular indeterminate color="accent" size="36" />
|
<v-progress-circular indeterminate color="accent" size="36" />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else-if="store.events.length === 0" class="fc-dl__empty">
|
<div v-else-if="filteredEvents.length === 0" class="fc-dl__empty">
|
||||||
<p>No download events match the current filter.</p>
|
<p>No download events match the current filter.</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else>
|
<div v-else>
|
||||||
<DownloadEventRow
|
<section
|
||||||
v-for="e in filteredEvents" :key="e.id" :event="e"
|
v-for="g in groups" :key="g.key"
|
||||||
@open="openDetail"
|
class="fc-dl__group"
|
||||||
/>
|
>
|
||||||
|
<header
|
||||||
|
class="fc-dl__group-head"
|
||||||
|
role="button" tabindex="0"
|
||||||
|
@click="toggle(g.key)" @keydown.enter="toggle(g.key)"
|
||||||
|
>
|
||||||
|
<v-icon size="small" class="fc-dl__group-chev">
|
||||||
|
{{ collapsed[g.key] ? 'mdi-chevron-right' : 'mdi-chevron-down' }}
|
||||||
|
</v-icon>
|
||||||
|
<span class="fc-dl__group-label">{{ g.label }}</span>
|
||||||
|
<span class="fc-dl__group-counts">
|
||||||
|
<v-chip
|
||||||
|
v-if="g.failedCount > 0"
|
||||||
|
size="x-small" color="error" variant="tonal"
|
||||||
|
prepend-icon="mdi-alert-circle"
|
||||||
|
>{{ g.failedCount }}</v-chip>
|
||||||
|
<v-chip size="x-small" variant="tonal">
|
||||||
|
{{ g.items.length }}
|
||||||
|
{{ g.items.length === 1 ? 'event' : 'events' }}
|
||||||
|
</v-chip>
|
||||||
|
</span>
|
||||||
|
</header>
|
||||||
|
<div v-if="!collapsed[g.key]" class="fc-dl__group-body">
|
||||||
|
<DownloadEventRow
|
||||||
|
v-for="e in g.items" :key="e.id" :event="e"
|
||||||
|
@open="openDetail"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
<div class="fc-dl__sentinel">
|
<div class="fc-dl__sentinel">
|
||||||
<v-btn v-if="store.hasMore" variant="text" @click="store.loadMore()" :loading="store.loading">
|
<v-btn v-if="store.hasMore" variant="text" @click="store.loadMore()" :loading="store.loading">
|
||||||
Load more
|
Load more
|
||||||
@@ -45,7 +74,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, ref, watch } from 'vue'
|
import { computed, onMounted, reactive, ref, watch } from 'vue'
|
||||||
import { useRoute } from 'vue-router'
|
import { useRoute } from 'vue-router'
|
||||||
|
|
||||||
import { useDownloadsStore } from '../../stores/downloads.js'
|
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||||
@@ -59,6 +88,16 @@ const route = useRoute()
|
|||||||
const store = useDownloadsStore()
|
const store = useDownloadsStore()
|
||||||
const filterModel = ref({ ...store.filter })
|
const filterModel = ref({ ...store.filter })
|
||||||
|
|
||||||
|
// Each group's collapsed state persists across refreshes for the
|
||||||
|
// lifetime of the SubscriptionsView (operator-friendly default: all
|
||||||
|
// expanded; collapse what you don't care about right now).
|
||||||
|
const collapsed = reactive({
|
||||||
|
today: false, yesterday: false, week: false, earlier: false,
|
||||||
|
})
|
||||||
|
function toggle(key) {
|
||||||
|
collapsed[key] = !collapsed[key]
|
||||||
|
}
|
||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
store.loadFirst(),
|
store.loadFirst(),
|
||||||
@@ -91,6 +130,48 @@ const filteredEvents = computed(() => {
|
|||||||
return arr
|
return arr
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Group events by relative date bucket and pin failed runs to the
|
||||||
|
// top of each bucket. Buckets boundaries are computed against the
|
||||||
|
// operator's local-time start-of-day so "Today" matches their
|
||||||
|
// intuition regardless of the event's stored UTC timestamp.
|
||||||
|
const groups = computed(() => {
|
||||||
|
const now = new Date()
|
||||||
|
const startOfToday = new Date(
|
||||||
|
now.getFullYear(), now.getMonth(), now.getDate(),
|
||||||
|
).getTime()
|
||||||
|
const startOfYesterday = startOfToday - 24 * 3600 * 1000
|
||||||
|
const startOfWeek = startOfToday - 7 * 24 * 3600 * 1000
|
||||||
|
|
||||||
|
const buckets = { today: [], yesterday: [], week: [], earlier: [] }
|
||||||
|
for (const e of filteredEvents.value) {
|
||||||
|
const t = new Date(e.started_at).getTime()
|
||||||
|
if (t >= startOfToday) buckets.today.push(e)
|
||||||
|
else if (t >= startOfYesterday) buckets.yesterday.push(e)
|
||||||
|
else if (t >= startOfWeek) buckets.week.push(e)
|
||||||
|
else buckets.earlier.push(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
function withFailedPinned(items) {
|
||||||
|
const fail = items.filter((e) => e.status === 'error')
|
||||||
|
const rest = items.filter((e) => e.status !== 'error')
|
||||||
|
return [...fail, ...rest]
|
||||||
|
}
|
||||||
|
|
||||||
|
const meta = [
|
||||||
|
{ key: 'today', label: 'Today' },
|
||||||
|
{ key: 'yesterday', label: 'Yesterday' },
|
||||||
|
{ key: 'week', label: 'Last 7 days' },
|
||||||
|
{ key: 'earlier', label: 'Earlier' },
|
||||||
|
]
|
||||||
|
return meta
|
||||||
|
.map(({ key, label }) => {
|
||||||
|
const items = withFailedPinned(buckets[key])
|
||||||
|
const failedCount = items.filter((e) => e.status === 'error').length
|
||||||
|
return { key, label, items, failedCount }
|
||||||
|
})
|
||||||
|
.filter((g) => g.items.length > 0)
|
||||||
|
})
|
||||||
|
|
||||||
watch(filterModel, async (m) => {
|
watch(filterModel, async (m) => {
|
||||||
await store.applyFilter({
|
await store.applyFilter({
|
||||||
status: m.status,
|
status: m.status,
|
||||||
@@ -120,4 +201,29 @@ async function openDetail(id) {
|
|||||||
.fc-dl__sentinel {
|
.fc-dl__sentinel {
|
||||||
display: flex; justify-content: center; padding: 1rem 0;
|
display: flex; justify-content: center; padding: 1rem 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.fc-dl__group { margin-bottom: 12px; }
|
||||||
|
.fc-dl__group-head {
|
||||||
|
display: flex; align-items: center; gap: 8px;
|
||||||
|
padding: 6px 8px;
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
user-select: none;
|
||||||
|
}
|
||||||
|
.fc-dl__group-head:hover {
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.08);
|
||||||
|
}
|
||||||
|
.fc-dl__group-chev {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-dl__group-label {
|
||||||
|
font-weight: 600;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.fc-dl__group-counts {
|
||||||
|
display: flex; gap: 6px; align-items: center;
|
||||||
|
}
|
||||||
|
.fc-dl__group-body { margin-top: 4px; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -146,6 +146,15 @@ export const useImportStore = defineStore('import', () => {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Layer-2 one-shot re-download for a failed task's (corrupt) file.
|
||||||
|
// Returns the endpoint's status dict (refetch_queued / no_source /
|
||||||
|
// already_refetched). Caller surfaces it as a toast.
|
||||||
|
async function refetchTask(taskId) {
|
||||||
|
const body = await api.post(`/api/import/tasks/${taskId}/refetch`)
|
||||||
|
await loadTasks(true)
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
const hasMore = computed(() => tasksNextCursor.value !== null)
|
const hasMore = computed(() => tasksNextCursor.value !== null)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -155,6 +164,7 @@ export const useImportStore = defineStore('import', () => {
|
|||||||
triggerError,
|
triggerError,
|
||||||
loadSettings, patchSettings,
|
loadSettings, patchSettings,
|
||||||
refreshStatus, triggerScan,
|
refreshStatus, triggerScan,
|
||||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck
|
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck,
|
||||||
|
refetchTask,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -171,6 +171,115 @@ async def test_trigger_still_rejects_unknown_mode(client):
|
|||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_refetch_404_for_unknown_task(client):
|
||||||
|
resp = await client.post("/api/import/tasks/999999/refetch")
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_refetch_400_for_non_failed_task(client, db):
|
||||||
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||||
|
db.add(batch)
|
||||||
|
await db.flush()
|
||||||
|
task = ImportTask(
|
||||||
|
batch_id=batch.id, source_path="/x.jpg", task_type="media",
|
||||||
|
status="complete", finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
db.add(task)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["status"] == "not_failed"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_refetch_no_source_when_unresolvable(client, db):
|
||||||
|
"""A failed task whose file has no sidecar / no resolvable Source
|
||||||
|
returns no_source (filesystem-only import — nothing to re-poll)."""
|
||||||
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||||
|
db.add(batch)
|
||||||
|
await db.flush()
|
||||||
|
task = ImportTask(
|
||||||
|
batch_id=batch.id, source_path="/import/nowhere/x.jpg",
|
||||||
|
task_type="media", status="failed", finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
db.add(task)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert (await resp.get_json())["status"] == "no_source"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_refetch_queued_with_resolvable_source(client, db, tmp_path, monkeypatch):
|
||||||
|
"""A failed task whose file resolves (via sidecar → artist+platform)
|
||||||
|
to an enabled, real-URL Source: the file is deleted, the task is
|
||||||
|
marked refetched, and ONE source re-check is queued."""
|
||||||
|
import json as _json
|
||||||
|
|
||||||
|
from sqlalchemy import update as _update
|
||||||
|
|
||||||
|
from backend.app.models import Artist, ImportSettings, Source
|
||||||
|
from backend.app.tasks import download as download_mod
|
||||||
|
|
||||||
|
# Stub the downloader so the eager test doesn't run a real fetch.
|
||||||
|
dispatched = []
|
||||||
|
monkeypatch.setattr(download_mod.download_source, "delay", dispatched.append)
|
||||||
|
|
||||||
|
# import_root/<ArtistName>/post.jpg + sidecar identifying the platform.
|
||||||
|
import_root = tmp_path / "import"
|
||||||
|
artist_dir = import_root / "Maewix"
|
||||||
|
artist_dir.mkdir(parents=True)
|
||||||
|
media = artist_dir / "post.jpg"
|
||||||
|
media.write_bytes(b"corrupt-bytes")
|
||||||
|
(artist_dir / "post.jpg.json").write_text(
|
||||||
|
_json.dumps({"category": "patreon", "post_id": 123})
|
||||||
|
)
|
||||||
|
|
||||||
|
# import_settings(id=1) is migration-seeded; point its scan path at
|
||||||
|
# our tmp import root rather than inserting a conflicting row.
|
||||||
|
await db.execute(
|
||||||
|
_update(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
.values(import_scan_path=str(import_root))
|
||||||
|
)
|
||||||
|
artist = Artist(name="Maewix", slug="maewix")
|
||||||
|
db.add(artist)
|
||||||
|
await db.flush()
|
||||||
|
db.add(Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/maewix", enabled=True,
|
||||||
|
config_overrides={},
|
||||||
|
))
|
||||||
|
batch = ImportBatch(triggered_by="manual", source_path=str(import_root), scan_mode="quick")
|
||||||
|
db.add(batch)
|
||||||
|
await db.flush()
|
||||||
|
task = ImportTask(
|
||||||
|
batch_id=batch.id, source_path=str(media), task_type="media",
|
||||||
|
status="failed", finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
db.add(task)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["status"] == "refetch_queued"
|
||||||
|
assert len(dispatched) == 1
|
||||||
|
assert not media.exists() # corrupt copy removed for re-fetch
|
||||||
|
|
||||||
|
from sqlalchemy import select as _select
|
||||||
|
refetched = (await db.execute(
|
||||||
|
_select(ImportTask.refetched).where(ImportTask.id == task.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert refetched is True
|
||||||
|
|
||||||
|
# Second attempt is a no-op (bounded to one).
|
||||||
|
resp2 = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||||
|
assert (await resp2.get_json())["status"] == "already_refetched"
|
||||||
|
assert len(dispatched) == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_trigger_accepts_verify(client, monkeypatch):
|
async def test_trigger_accepts_verify(client, monkeypatch):
|
||||||
# Stub the verify task's dispatch so the API contract is asserted
|
# Stub the verify task's dispatch so the API contract is asserted
|
||||||
|
|||||||
+138
-3
@@ -164,6 +164,52 @@ def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch
|
|||||||
assert dispatched == [stuck.id] # stuck rows re-enqueue; orphans don't
|
assert dispatched == [stuck.id] # stuck rows re-enqueue; orphans don't
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_interrupted_poison_pill_caps_at_max(db_sync, monkeypatch):
|
||||||
|
"""A stuck row that's already been recovered MAX_RECOVERY_ATTEMPTS-1
|
||||||
|
times is marked 'failed' (with a diagnostic) instead of re-queued —
|
||||||
|
the circuit breaker against an input that hard-crashes the worker
|
||||||
|
every run. Operator-flagged 2026-05-28."""
|
||||||
|
from backend.app.tasks import import_file
|
||||||
|
from backend.app.tasks.maintenance import (
|
||||||
|
MAX_RECOVERY_ATTEMPTS,
|
||||||
|
recover_interrupted_tasks,
|
||||||
|
)
|
||||||
|
dispatched: list[int] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
import_file.import_media_file, "delay", dispatched.append
|
||||||
|
)
|
||||||
|
|
||||||
|
batch_id = _make_batch(db_sync)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
|
||||||
|
# At the cap already (recovered MAX-1 times) → fail, don't re-queue.
|
||||||
|
poison = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/poison.jpg", task_type="media",
|
||||||
|
status="processing", started_at=now - timedelta(hours=2),
|
||||||
|
recovery_count=MAX_RECOVERY_ATTEMPTS - 1,
|
||||||
|
)
|
||||||
|
# One recovery short of the cap → re-queue + increment.
|
||||||
|
recoverable = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/ok.jpg", task_type="media",
|
||||||
|
status="processing", started_at=now - timedelta(hours=2),
|
||||||
|
recovery_count=MAX_RECOVERY_ATTEMPTS - 2,
|
||||||
|
)
|
||||||
|
db_sync.add_all([poison, recoverable])
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
touched = recover_interrupted_tasks.apply().get()
|
||||||
|
assert touched == 2 # one failed + one re-queued
|
||||||
|
|
||||||
|
db_sync.refresh(poison)
|
||||||
|
db_sync.refresh(recoverable)
|
||||||
|
assert poison.status == "failed"
|
||||||
|
assert "corrupt or" in (poison.error or "")
|
||||||
|
assert recoverable.status == "queued"
|
||||||
|
assert recoverable.recovery_count == MAX_RECOVERY_ATTEMPTS - 1
|
||||||
|
# Only the recoverable row re-enqueues; the poison pill does not.
|
||||||
|
assert dispatched == [recoverable.id]
|
||||||
|
|
||||||
|
|
||||||
def test_cleanup_old_deletes_finished_old(db_sync):
|
def test_cleanup_old_deletes_finished_old(db_sync):
|
||||||
batch_id = _make_batch(db_sync)
|
batch_id = _make_batch(db_sync)
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
@@ -195,12 +241,13 @@ def test_cleanup_old_deletes_finished_old(db_sync):
|
|||||||
|
|
||||||
|
|
||||||
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
|
def _make_task_run(db_sync, *, status, started_at, finished_at=None,
|
||||||
error_type=None):
|
error_type=None, queue="default",
|
||||||
|
task_name="backend.app.tasks.fake.t"):
|
||||||
from backend.app.models import TaskRun
|
from backend.app.models import TaskRun
|
||||||
row = TaskRun(
|
row = TaskRun(
|
||||||
celery_task_id="x",
|
celery_task_id="x",
|
||||||
queue="ml",
|
queue=queue,
|
||||||
task_name="backend.app.tasks.fake.t",
|
task_name=task_name,
|
||||||
target_id=1,
|
target_id=1,
|
||||||
started_at=started_at,
|
started_at=started_at,
|
||||||
finished_at=finished_at,
|
finished_at=finished_at,
|
||||||
@@ -262,6 +309,94 @@ def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
|
|||||||
assert status == "running"
|
assert status == "running"
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_stalled_task_runs_ml_queue_uses_longer_threshold(db_sync):
|
||||||
|
"""ml-queue tasks (tag_and_embed video branch) legitimately run
|
||||||
|
past the default 5-min threshold. The sweep must NOT flag an
|
||||||
|
ml-queue task that's only been running 10 min — the override
|
||||||
|
threshold (25 min via QUEUE_STUCK_THRESHOLD_MINUTES) protects
|
||||||
|
in-flight video tagging. Operator-flagged 2026-05-28 after
|
||||||
|
image 6288 (mp4) was marked failed at the 5-min tick mid-run."""
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
# 10-min-old ml-queue row: stale by the default 5-min rule but
|
||||||
|
# fresh by the 25-min ml override. Must survive the sweep.
|
||||||
|
ml_fresh_id = _make_task_run(
|
||||||
|
db_sync, status="running", queue="ml",
|
||||||
|
started_at=now - timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
# 30-min-old ml-queue row: past even the ml override. Must be
|
||||||
|
# flagged.
|
||||||
|
ml_stale_id = _make_task_run(
|
||||||
|
db_sync, status="running", queue="ml",
|
||||||
|
started_at=now - timedelta(minutes=30),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
recovered = recover_stalled_task_runs.apply().get()
|
||||||
|
assert recovered == 1
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
ml_fresh_status = db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.id == ml_fresh_id)
|
||||||
|
).scalar_one()
|
||||||
|
ml_stale_status = db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.id == ml_stale_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert ml_fresh_status == "running"
|
||||||
|
assert ml_stale_status == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_stalled_task_runs_archive_task_uses_longer_threshold(db_sync):
|
||||||
|
"""import_archive_file shares the 'import' queue with fast
|
||||||
|
single-file import_media_file, so it gets a per-task-name override
|
||||||
|
(40 min) while the import queue stays at the 5-min default. A
|
||||||
|
10-min-old archive task-run must survive; a 50-min-old one is
|
||||||
|
flagged. Operator-flagged 2026-05-28."""
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
from backend.app.tasks.maintenance import recover_stalled_task_runs
|
||||||
|
|
||||||
|
archive_name = "backend.app.tasks.import_file.import_archive_file"
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
# Fast single-file import on the same queue, 10 min old → flagged
|
||||||
|
# by the default 5-min rule.
|
||||||
|
media_id = _make_task_run(
|
||||||
|
db_sync, status="running", queue="import",
|
||||||
|
task_name="backend.app.tasks.import_file.import_media_file",
|
||||||
|
started_at=now - timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
# Archive on the same queue, 10 min old → survives (40-min override).
|
||||||
|
archive_fresh_id = _make_task_run(
|
||||||
|
db_sync, status="running", queue="import",
|
||||||
|
task_name=archive_name,
|
||||||
|
started_at=now - timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
# Archive 50 min old → past even the 40-min override → flagged.
|
||||||
|
archive_stale_id = _make_task_run(
|
||||||
|
db_sync, status="running", queue="import",
|
||||||
|
task_name=archive_name,
|
||||||
|
started_at=now - timedelta(minutes=50),
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
recovered = recover_stalled_task_runs.apply().get()
|
||||||
|
assert recovered == 2 # media + stale archive
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
def _status(_id):
|
||||||
|
return db_sync.execute(
|
||||||
|
select(TaskRun.status).where(TaskRun.id == _id)
|
||||||
|
).scalar_one()
|
||||||
|
assert _status(media_id) == "error"
|
||||||
|
assert _status(archive_fresh_id) == "running"
|
||||||
|
assert _status(archive_stale_id) == "error"
|
||||||
|
|
||||||
|
|
||||||
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
|
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""Layer-3 subprocess-isolated probe tests.
|
||||||
|
|
||||||
|
The bomb-guard cap is exercised against `_archive_probe_target` directly
|
||||||
|
(in-process, where a monkeypatch on the module constant takes effect) —
|
||||||
|
spawn re-imports the module in the child, so a parent-process
|
||||||
|
monkeypatch wouldn't reach the spawned worker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import multiprocessing as mp
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from backend.app.utils import safe_probe
|
||||||
|
|
||||||
|
|
||||||
|
def _zip(path, entries):
|
||||||
|
with zipfile.ZipFile(path, "w") as zf:
|
||||||
|
for name, data in entries.items():
|
||||||
|
zf.writestr(name, data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_archive_valid_zip(tmp_path):
|
||||||
|
z = tmp_path / "ok.zip"
|
||||||
|
_zip(z, {"a.jpg": b"hello", "b.png": b"world"})
|
||||||
|
res = safe_probe.probe_archive(z)
|
||||||
|
assert res.ok is True
|
||||||
|
assert res.crashed is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_archive_corrupt_zip_clean_rejection(tmp_path):
|
||||||
|
z = tmp_path / "broken.zip"
|
||||||
|
z.write_bytes(b"PK\x03\x04 not really a zip past here")
|
||||||
|
res = safe_probe.probe_archive(z)
|
||||||
|
assert res.ok is False
|
||||||
|
# Corrupt-but-handled (zipfile raises BadZipFile in the child) — a
|
||||||
|
# clean rejection, not a hard crash.
|
||||||
|
assert res.crashed is False
|
||||||
|
assert res.reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
|
||||||
|
z = tmp_path / "sized.zip"
|
||||||
|
_zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50})
|
||||||
|
total, bad = safe_probe._inspect_archive(z, ".zip")
|
||||||
|
assert total == 150
|
||||||
|
assert bad is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_archive_probe_target_bomb_guard(tmp_path, monkeypatch):
|
||||||
|
"""In-process call to the child target so the monkeypatched cap
|
||||||
|
takes effect. A normal zip whose uncompressed size exceeds the
|
||||||
|
(lowered) cap is rejected with the bomb-guard reason."""
|
||||||
|
monkeypatch.setattr(safe_probe, "MAX_ARCHIVE_UNCOMPRESSED_BYTES", 10)
|
||||||
|
z = tmp_path / "bomb.zip"
|
||||||
|
_zip(z, {"big.txt": b"x" * 5000}) # 5000 uncompressed > 10-byte cap
|
||||||
|
q = mp.get_context("spawn").Queue()
|
||||||
|
safe_probe._archive_probe_target(str(z), q)
|
||||||
|
status, detail = q.get(timeout=5)
|
||||||
|
assert status == "error"
|
||||||
|
assert "bomb-guard cap" in detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_video_non_video_is_not_ok(tmp_path):
|
||||||
|
"""A text file is not a decodable video. Whether ffprobe is present
|
||||||
|
(returncode != 0) or absent (OSError → 'unavailable'), the result is
|
||||||
|
ok=False. We don't assert on crashed/reason so the test is robust to
|
||||||
|
ffprobe presence in CI."""
|
||||||
|
f = tmp_path / "nope.txt"
|
||||||
|
f.write_text("definitely not a video container")
|
||||||
|
res = safe_probe.probe_video(f)
|
||||||
|
assert res.ok is False
|
||||||
@@ -21,6 +21,10 @@ def test_import_media_file_registered():
|
|||||||
assert "backend.app.tasks.import_file.import_media_file" in celery.tasks
|
assert "backend.app.tasks.import_file.import_media_file" in celery.tasks
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_archive_file_registered():
|
||||||
|
assert "backend.app.tasks.import_file.import_archive_file" in celery.tasks
|
||||||
|
|
||||||
|
|
||||||
def test_generate_thumbnail_registered():
|
def test_generate_thumbnail_registered():
|
||||||
assert "backend.app.tasks.thumbnail.generate_thumbnail" in celery.tasks
|
assert "backend.app.tasks.thumbnail.generate_thumbnail" in celery.tasks
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user