Admin IA overhaul, cap-aware autoscaler, optional ml-worker, hardened tag reset #187
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
|
||||
# Bump on every agent change. The page embeds this and /status reports it; the UI
|
||||
# warns to reload when they differ — so a stale browser-cached page can't be
|
||||
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
|
||||
VERSION = "2026-07-02.4 · bandwidth governor: aggregate download cap (MB/s dial) so the agent can't saturate the desktop's network"
|
||||
VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
|
||||
|
||||
logbuf.install()
|
||||
cfg = Config.from_env()
|
||||
@@ -344,7 +344,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
// Auto on → dial reflects the auto-chosen count (read-only); off → manual.
|
||||
if(document.activeElement!==autochk) autochk.checked=!!s.auto
|
||||
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
|
||||
conchint.textContent=s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP)
|
||||
conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
|
||||
+(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
|
||||
if(document.activeElement!==conc) conc.value=s.concurrency
|
||||
conc.max=CAP
|
||||
// Connection pill + queue come only from the /status poll (the Start/Stop POST
|
||||
|
||||
@@ -124,6 +124,17 @@ OCC_LOW = 0.25 # below this = buffer starving → add a downloade
|
||||
OCC_HIGH = 0.80 # above this = downloaders outpace the GPU
|
||||
UTIL_ALPHA = 0.25 # GPU-util EWMA weight
|
||||
UTIL_START = 85 # GPU has headroom below this (gate a 2nd consumer)
|
||||
# Bandwidth-cap awareness (operator 2026-07-02): with the aggregate governor in
|
||||
# place, the occupancy signal alone would peg downloaders at DL_MAX while the
|
||||
# CAP — not concurrency — is the real constraint: 8 streams sharing 8 MB/s move
|
||||
# no more data than 4, they just hold more leases + RAM and stretch every job's
|
||||
# latency. So growth is gated on the pipe having headroom, and a pipe pinned at
|
||||
# the cap sheds streams down to BW_MIN_DL (enough overlap to keep the cap
|
||||
# filled through TTFB + decode gaps). The dead band between the two thresholds
|
||||
# prevents add/trim flapping.
|
||||
BW_ADD_HEADROOM = 0.85 # add a downloader only while net < 85% of the cap
|
||||
BW_TRIM_AT = 0.95 # net ≥ 95% of the cap → shed toward BW_MIN_DL
|
||||
BW_MIN_DL = 3
|
||||
VRAM_HI = 0.90 # memory pressure → shed a consumer
|
||||
VRAM_GROW_MAX = 0.82 # don't add a consumer past this VRAM
|
||||
TPUT_ALPHA = 0.5 # throughput EWMA weight
|
||||
@@ -225,6 +236,7 @@ class Worker:
|
||||
self._jpm = 0.0
|
||||
self._dpm = 0.0
|
||||
self._net_mb_s = 0.0 # smoothed aggregate download rate (UI readout)
|
||||
self._bw_capped = False # autoscaler is holding/shedding at the cap (UI)
|
||||
self._util_smooth: float | None = None # EWMA GPU util (set by control loop)
|
||||
# Curator queue snapshot, refreshed by a background poller so the UI
|
||||
# /status read is instant — never an inline curator HTTP call (which
|
||||
@@ -584,6 +596,7 @@ class Worker:
|
||||
"transient": self.transient,
|
||||
"bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
|
||||
"net_mb_s": round(self._net_mb_s, 1), # observed aggregate rate
|
||||
"bw_capped": self._bw_capped, # autoscaler holding at the cap (UI hint)
|
||||
}
|
||||
|
||||
def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
|
||||
@@ -982,10 +995,19 @@ class Worker:
|
||||
tick = 0
|
||||
con_grew = False
|
||||
self._util_smooth = None
|
||||
self._bw_capped = False
|
||||
continue
|
||||
|
||||
occ = self._buffer.qsize() / BUFFER_MAX
|
||||
occ_ewma = _ewma(occ_ewma, occ, OCC_ALPHA)
|
||||
# Bandwidth-cap position: compare the observed aggregate (the same
|
||||
# EWMA the UI shows) against the governor's cap. `soft` gates
|
||||
# growth; `hard` sheds streams (see BW_* rationale above).
|
||||
bw_rate = self.throttle.rate
|
||||
net_bytes = self._net_mb_s * 1_048_576
|
||||
bw_soft = bw_rate > 0 and net_bytes >= BW_ADD_HEADROOM * bw_rate
|
||||
bw_hard = bw_rate > 0 and net_bytes >= BW_TRIM_AT * bw_rate
|
||||
self._bw_capped = bw_soft
|
||||
g = gpumod.read_gpu() or {}
|
||||
mt = g.get("mem_total_mb") or 0
|
||||
vram = (g.get("mem_used_mb", 0) / mt) if mt else 0.0
|
||||
@@ -1022,8 +1044,15 @@ class Worker:
|
||||
self._apply_downloaders(-1)
|
||||
con_grew = False
|
||||
elif occ_ewma < OCC_LOW:
|
||||
# Buffer starving → GPU idle waiting on downloads → add a feeder.
|
||||
self._apply_downloaders(+1)
|
||||
# Buffer starving → downloads are the bottleneck. WHICH kind
|
||||
# decides the move: a pipe pinned at the bandwidth cap gains
|
||||
# nothing from more streams (they'd split the same budget and
|
||||
# stretch per-job latency) — shed toward BW_MIN_DL; with cap
|
||||
# headroom, concurrency is genuinely short — add a feeder.
|
||||
if bw_hard and self._dl_target > BW_MIN_DL:
|
||||
self._apply_downloaders(-1)
|
||||
elif not bw_soft:
|
||||
self._apply_downloaders(+1)
|
||||
con_grew = False
|
||||
elif occ_ewma > OCC_HIGH:
|
||||
# Downloaders outpace the GPU. Prefer helping the GPU (add a 2nd
|
||||
@@ -1049,7 +1078,9 @@ class Worker:
|
||||
if self._dl_target != d0 or self._consumer_target != c0:
|
||||
log.info(
|
||||
"autoscale: dl %d→%d · consumers %d→%d "
|
||||
"(buf %d%% · util~%d%% · %.2f j/s · vram %d%%)",
|
||||
"(buf %d%% · util~%d%% · %.2f j/s · vram %d%% · "
|
||||
"net %.1f MB/s%s)",
|
||||
d0, self._dl_target, c0, self._consumer_target,
|
||||
round(occ_ewma * 100), round(util_ewma), tput_ewma,
|
||||
round(vram * 100))
|
||||
round(vram * 100), self._net_mb_s,
|
||||
" — at cap" if bw_soft else "")
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch
|
||||
|
||||
B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU
|
||||
whole-image embed for stacks without a GPU agent. ON by default (a fresh
|
||||
install works agent-less); agent-equipped stacks that drop the ml-worker
|
||||
container turn it off so import hooks stop queueing embed work into a queue
|
||||
nothing consumes — the daily GPU 'embed' backfill covers those images.
|
||||
|
||||
Revision ID: 0074
|
||||
Revises: 0073
|
||||
Create Date: 2026-07-02
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0074"
|
||||
down_revision: Union[str, None] = "0073"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"cpu_embed_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.true(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("ml_settings", "cpu_embed_enabled")
|
||||
@@ -276,18 +276,48 @@ async def posts_reconcile_duplicates():
|
||||
return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
|
||||
|
||||
|
||||
def _reset_content_confirm_token(projection: dict) -> str:
|
||||
"""Stable 8-hex token derived from the live counts (mirrors the Tier-C
|
||||
bulk-delete token): it changes whenever the data changes, so the apply can
|
||||
only ever run against numbers the operator just previewed."""
|
||||
canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
|
||||
return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
@admin_bp.route("/tags/reset-content", methods=["POST"])
|
||||
async def tags_reset_content():
|
||||
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
|
||||
content vocabulary) so the operator can re-tag from scratch via
|
||||
auto-suggest. fandom + series tags + series_page ordering are preserved,
|
||||
and image_prediction rows are untouched so suggestions repopulate.
|
||||
dry-run preview returns per-kind counts + applications + a sample so the
|
||||
UI shows exactly what'll go before the operator confirms (dry_run=false).
|
||||
Irreversible except via DB backup restore."""
|
||||
"""Full-instance reset of the CONTENT vocabulary: deletes ALL general +
|
||||
character tags and their image applications — INCLUDING the examples the
|
||||
tagging heads learned from. Suggestions do NOT repopulate on their own
|
||||
(the Camie predictions that once did are long retired): the operator
|
||||
re-tags from scratch and the heads retrain from the new signal. fandom +
|
||||
series tags + series_page ordering are preserved.
|
||||
|
||||
Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
|
||||
the full reset stays, but behind extra steps): dry_run returns the
|
||||
projection + a `confirm` token derived from the live counts; the apply
|
||||
must echo that token back or it is rejected."""
|
||||
from ..services.cleanup_service import reset_content_tagging
|
||||
|
||||
return await _run_dry_run_op(reset_content_tagging)
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
async with get_session() as session:
|
||||
projection = await session.run_sync(
|
||||
lambda s: reset_content_tagging(s, dry_run=True)
|
||||
)
|
||||
token = _reset_content_confirm_token(projection)
|
||||
if dry_run:
|
||||
projection["confirm"] = token
|
||||
return jsonify(projection)
|
||||
if str(body.get("confirm", "")) != token:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail="run a fresh preview and echo its confirm token",
|
||||
)
|
||||
result = await session.run_sync(
|
||||
lambda s: reset_content_tagging(s, dry_run=False)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/normalize", methods=["POST"])
|
||||
|
||||
@@ -96,7 +96,7 @@ async def backfill():
|
||||
"""Enqueue a job for every image that doesn't already have one for `task`."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
task = str(body.get("task") or "ccip")
|
||||
from ..tasks.ml import enqueue_gpu_backfill
|
||||
from ..tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
r = enqueue_gpu_backfill.delay(task)
|
||||
return jsonify({"celery_task_id": r.id, "task": task}), 202
|
||||
@@ -109,7 +109,7 @@ async def reprocess():
|
||||
detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
task = str(body.get("task") or "ccip")
|
||||
from ..tasks.ml import reprocess_gpu_jobs
|
||||
from ..tasks.gpu_queue import reprocess_gpu_jobs
|
||||
|
||||
r = reprocess_gpu_jobs.delay(task)
|
||||
return jsonify({"celery_task_id": r.id, "task": task}), 202
|
||||
|
||||
@@ -9,6 +9,7 @@ ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||
|
||||
|
||||
_EDITABLE = (
|
||||
"cpu_embed_enabled",
|
||||
"video_frame_interval_seconds",
|
||||
"video_max_frames",
|
||||
"head_min_positives",
|
||||
@@ -63,6 +64,7 @@ async def get_settings():
|
||||
).scalar_one()
|
||||
return jsonify(
|
||||
{
|
||||
"cpu_embed_enabled": s.cpu_embed_enabled,
|
||||
"video_frame_interval_seconds": s.video_frame_interval_seconds,
|
||||
"video_max_frames": s.video_max_frames,
|
||||
"embedder_model_version": s.embedder_model_version,
|
||||
|
||||
@@ -7,7 +7,7 @@ Queues:
|
||||
download — gallery-dl tasks (FC-3)
|
||||
scan — periodic source checks (FC-3) — kept separate so long imports
|
||||
don't starve the scheduler
|
||||
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
||||
maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
|
||||
default — anything not explicitly routed
|
||||
"""
|
||||
|
||||
@@ -29,6 +29,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.thumbnail",
|
||||
"backend.app.tasks.maintenance",
|
||||
"backend.app.tasks.ml",
|
||||
"backend.app.tasks.gpu_queue",
|
||||
"backend.app.tasks.download",
|
||||
"backend.app.tasks.external",
|
||||
"backend.app.tasks.backup",
|
||||
@@ -41,6 +42,11 @@ def make_celery() -> Celery:
|
||||
task_routes={
|
||||
"backend.app.tasks.import_file.*": {"queue": "import"},
|
||||
"backend.app.tasks.ml.*": {"queue": "ml"},
|
||||
# GPU-queue coordination (backfill enqueues, orphan recovery,
|
||||
# reprocess) is pure DB work — it rides the maintenance quick lane
|
||||
# so the GPU agent pipeline works even on stacks that drop the
|
||||
# (now-optional, B3) ml-worker container entirely.
|
||||
"backend.app.tasks.gpu_queue.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
||||
"backend.app.tasks.download.*": {"queue": "download"},
|
||||
# External file-host fetches are downloads — same lane (they can run
|
||||
@@ -106,7 +112,7 @@ def make_celery() -> Celery:
|
||||
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
||||
},
|
||||
"recover-orphaned-gpu-jobs": {
|
||||
"task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
|
||||
"task": "backend.app.tasks.gpu_queue.recover_orphaned_gpu_jobs",
|
||||
"schedule": 60.0, # quick pickup of work a dead agent orphaned
|
||||
},
|
||||
"triage-gpu-errors": {
|
||||
@@ -114,17 +120,17 @@ def make_celery() -> Celery:
|
||||
"schedule": 900.0, # probe errored jobs' files → defect/file_ok
|
||||
},
|
||||
"enqueue-ccip-backfill-hourly": {
|
||||
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
||||
"task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
|
||||
"schedule": 3600.0, # auto-feed NEW images; errored are
|
||||
"args": ("ccip",), # tombstoned — retry is the button only
|
||||
},
|
||||
"enqueue-siglip-backfill-daily": {
|
||||
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
||||
"task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
|
||||
"schedule": 86400.0, # drain the concept-crop back-catalogue
|
||||
"args": ("siglip",), # (errored are tombstoned, not retried)
|
||||
},
|
||||
"enqueue-embed-backfill-daily": {
|
||||
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
||||
"task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
|
||||
"schedule": 86400.0, # whole-image re-embed under the current
|
||||
"args": ("embed",), # model (an operator swap) drains via agent
|
||||
},
|
||||
|
||||
@@ -23,6 +23,15 @@ class MLSettings(Base):
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# CPU whole-image embedding (B3, operator 2026-07-02). The ml-worker's ONLY
|
||||
# processing role is the embed fallback for stacks WITHOUT a GPU agent — ON
|
||||
# by default so a fresh install works with no agent. Stacks that run the
|
||||
# agent and drop the ml-worker container turn this OFF so import hooks stop
|
||||
# queueing embed work nothing will consume (the daily GPU 'embed' backfill
|
||||
# covers those images instead).
|
||||
cpu_embed_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
# Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
|
||||
# a fixed count) so coverage reflects real screen time regardless of length;
|
||||
# cap the total so a long video can't explode into hundreds of embeds. The
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Single-color audit: matches images where one color dominates beyond
|
||||
the threshold (within the given Euclidean RGB tolerance). The first
|
||||
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||
future spec can adopt it on the import path too.
|
||||
the threshold (within the given Euclidean RGB tolerance). The canonical
|
||||
predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||
SkipReason.single_color), so what the audit flags and what the import
|
||||
skips can never disagree.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -726,7 +726,10 @@ RESETTABLE_TAG_KINDS = ("general", "character")
|
||||
|
||||
def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
|
||||
"""Count (dry_run) or DELETE every general + character tag so the operator
|
||||
can re-tag from scratch (heads/CCIP repopulate suggestions).
|
||||
can re-tag from scratch. NB: the deleted applications include the tagging
|
||||
heads' training positives — suggestions do NOT repopulate on their own; the
|
||||
heads retrain from whatever the operator re-tags. (The API route gates the
|
||||
live run behind a preview-derived confirm token for exactly this reason.)
|
||||
|
||||
PRESERVED: fandom + series tags and their series_page ordering. CASCADE on
|
||||
image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
|
||||
@@ -1005,7 +1008,7 @@ def reextract_archive_attachments(
|
||||
still an archive on disk, so the cursor is what guarantees forward progress.
|
||||
"""
|
||||
from ..models import ImportSettings, Post, PostAttachment, Source
|
||||
from ..tasks.ml import tag_and_embed
|
||||
from ..tasks.ml import cpu_embed_enabled, embed_image
|
||||
from ..tasks.thumbnail import generate_thumbnail
|
||||
from .archive_extractor import is_archive
|
||||
from .importer import Importer
|
||||
@@ -1086,10 +1089,12 @@ def reextract_archive_attachments(
|
||||
|
||||
# Thumbnails + ML for the newly-imported members (best-effort; off the
|
||||
# critical path — a Redis hiccup must not fail the whole re-extract).
|
||||
do_embed = cpu_embed_enabled()
|
||||
for img_id in enqueue_ids:
|
||||
try:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
if do_embed:
|
||||
embed_image.delay(img_id)
|
||||
except Exception as exc:
|
||||
log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
|
||||
return summary
|
||||
|
||||
@@ -326,14 +326,16 @@ class DownloadService:
|
||||
# for hours after a download landed. Lazy import to avoid
|
||||
# circular-import risk between this service and the
|
||||
# tasks/* modules that import it.
|
||||
from ..tasks.ml import tag_and_embed
|
||||
from ..tasks.ml import cpu_embed_enabled, embed_image
|
||||
from ..tasks.thumbnail import generate_thumbnail
|
||||
do_embed = cpu_embed_enabled()
|
||||
ids = list(result.member_image_ids)
|
||||
if result.image_id is not None and result.image_id not in ids:
|
||||
ids.append(result.image_id)
|
||||
for img_id in ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
if do_embed:
|
||||
embed_image.delay(img_id)
|
||||
elif result.status == "attached":
|
||||
# Non-media or extracted archive captured as PostAttachment
|
||||
# (FC-2d-iii). The canonical copy lives in the attachments
|
||||
|
||||
@@ -44,6 +44,7 @@ from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from ..utils.slug import slugify
|
||||
from .archive_extractor import extract_archive, is_archive
|
||||
from .attachment_store import AttachmentStore
|
||||
from .audits import single_color
|
||||
from .link_extract import extract_external_links
|
||||
from .thumbnailer import Thumbnailer
|
||||
|
||||
@@ -790,6 +791,13 @@ class Importer:
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(source):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
|
||||
# Artist anchored to the attribution path (folder→artist), resolved
|
||||
# UP-FRONT so the enrich-on-duplicate branches link provenance with the
|
||||
# right artist even when the sidecar carries none — which is now the norm
|
||||
@@ -1123,6 +1131,13 @@ class Importer:
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(path):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
else:
|
||||
# Best-effort probe for dims + duration so downloaded videos can dedup
|
||||
# (#871). LENIENT: unlike _import_media this path does not reject on a
|
||||
@@ -1538,6 +1553,33 @@ class Importer:
|
||||
# Benign orphan; the DB swap already committed. Don't undo it.
|
||||
pass
|
||||
|
||||
# Matches the Cleanup audit card's default tolerance: the import-side
|
||||
# filter and the retroactive audit must agree on what "single color" MEANS
|
||||
# (Euclidean RGB distance to the dominant color); only the match threshold
|
||||
# is operator-tunable per surface.
|
||||
_SINGLE_COLOR_TOLERANCE = 30
|
||||
|
||||
def _single_color_hit(self, source: Path) -> bool:
|
||||
"""True when one color dominates beyond the configured threshold — the
|
||||
same canonical predicate the Cleanup audit runs (audits.single_color,
|
||||
whose docstring anticipated this adoption; the skip_single_color
|
||||
setting existed but was never wired until 2026-07-02). Never raises:
|
||||
unreadable files were already rejected by verify() upstream, and a
|
||||
residual decode error just declines to match (the import proceeds)."""
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
if getattr(im, "is_animated", False):
|
||||
# Frame 0 only would misjudge animations; skip like the
|
||||
# transparency check does.
|
||||
return False
|
||||
return single_color.evaluate(
|
||||
im,
|
||||
threshold=self.settings.single_color_threshold,
|
||||
tolerance=self._SINGLE_COLOR_TOLERANCE,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _transparency_pct(self, source: Path) -> float:
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases."""
|
||||
"""ML pipeline services: embedders, heads (the learning suggester), suggestions,
|
||||
GPU-job queue + failure triage, CCIP characters, crops/regions, allowlist and
|
||||
aliases."""
|
||||
|
||||
@@ -216,11 +216,13 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
# Thumbnails + ML for any newly-attached images (mirrors the download
|
||||
# path). Lazy import to dodge a task-module import cycle.
|
||||
if image_ids:
|
||||
from .ml import tag_and_embed
|
||||
from .ml import cpu_embed_enabled, embed_image
|
||||
from .thumbnail import generate_thumbnail
|
||||
do_embed = cpu_embed_enabled()
|
||||
for img_id in image_ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
if do_embed:
|
||||
embed_image.delay(img_id)
|
||||
return {"link_id": link_id, "files": len(result.files), "images": len(image_ids)}
|
||||
except Exception as exc: # never leave a link stuck in 'downloading'
|
||||
log.exception("external fetch task failed for link %s", link_id)
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
"""GPU-job queue coordination: backfill enqueues, orphan recovery, reprocess.
|
||||
|
||||
These are pure-DB sweeps (INSERT…SELECT / UPDATE) — no torch, no sklearn —
|
||||
that keep the desktop GPU agent's work queue fed and self-healing. They lived
|
||||
in tasks/ml.py (routed to the 'ml' queue) purely by colocation, which made the
|
||||
ml-worker container a hard dependency of the GPU pipeline; under B3 the
|
||||
ml-worker is OPTIONAL (its only processing role is the CPU embed fallback), so
|
||||
these moved here and route to the 'maintenance' quick lane with the other
|
||||
recovery sweeps. A stack with no ml-worker keeps a fully-working GPU pipeline.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..celery_app import celery
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.gpu_queue.enqueue_gpu_backfill")
|
||||
def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
"""Enqueue a gpu_job for every image that still needs `task_name` (one
|
||||
INSERT…SELECT, so it scales to a full library). The desktop agent drains the
|
||||
queue over HTTP. Returns the number enqueued.
|
||||
|
||||
Completion is judged PER PIPELINE, never across them (B3, operator
|
||||
2026-07-02): 'ccip' by prior gpu_job rows, 'siglip' by concept regions at
|
||||
the current model version, and only 'embed' by image_record's whole-image
|
||||
embedding — the one artifact the CPU fallback also produces. A CPU embed
|
||||
therefore never closes crop/detect work for the agent.
|
||||
|
||||
An ERRORED job is a tombstone for its (image, task): no variant re-enqueues
|
||||
it. Retry is deliberate-only (/retry_errors), which also means an errored
|
||||
back-catalogue needs one "Retry errored jobs" press after a model swap.
|
||||
Before the tombstone rule, this loop re-minted a fresh doomed job for every
|
||||
permanently-bad file each run — ~24 duplicate error rows/day per file (the
|
||||
2026-07-02 "unprocessable" flood)."""
|
||||
from sqlalchemy import exists, insert, literal, or_
|
||||
from sqlalchemy import select as sa_select
|
||||
|
||||
from ..models import GpuJob, ImageRecord, ImageRegion, MLSettings
|
||||
from ..services.ml.gpu_jobs import error_dedupe_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
# Prune stale tombstones first (loop-era duplicates + rows made moot by
|
||||
# a later success), so 'error' reads as one row per distinct failing
|
||||
# file and the skip-guards below see a clean picture.
|
||||
pruned = sum(
|
||||
session.execute(s).rowcount or 0 for s in error_dedupe_statements()
|
||||
)
|
||||
if pruned:
|
||||
log.info("gpu backfill: pruned %d stale/duplicate error rows", pruned)
|
||||
cur_version = session.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
).scalar_one()
|
||||
if task_name == "embed":
|
||||
# Whole-image GPU re-embed (#1190): images with no embedding, or one
|
||||
# stamped under a DIFFERENT model version (an operator model swap).
|
||||
stale = or_(
|
||||
ImageRecord.siglip_embedding.is_(None),
|
||||
ImageRecord.siglip_model_version.is_(None),
|
||||
ImageRecord.siglip_model_version != cur_version,
|
||||
)
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "embed",
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("embed"), literal("pending")
|
||||
).where(stale).where(~blocked)
|
||||
elif task_name == "siglip":
|
||||
# Concept-crop re-embed: enqueue when there's no concept region AT THE
|
||||
# CURRENT model version — so a model swap re-triggers crops too, not
|
||||
# only the never-embedded back-catalogue.
|
||||
has_current_concept = exists().where(
|
||||
ImageRegion.image_record_id == ImageRecord.id,
|
||||
ImageRegion.kind == "concept",
|
||||
ImageRegion.embedding_version == cur_version,
|
||||
)
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "siglip",
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("siglip"), literal("pending")
|
||||
).where(~has_current_concept).where(~blocked)
|
||||
else:
|
||||
# ANY prior row blocks — including 'error' (tombstone rule, see
|
||||
# docstring): pre-fix this branch ran HOURLY and was the loop.
|
||||
already = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == task_name,
|
||||
GpuJob.status.in_(["pending", "leased", "done", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal(task_name), literal("pending")
|
||||
).where(~already)
|
||||
# RETURNING + count: result.rowcount is unreliable for INSERT…SELECT.
|
||||
rows = session.execute(
|
||||
insert(GpuJob)
|
||||
.from_select(["image_record_id", "task", "status"], sel)
|
||||
.returning(GpuJob.id)
|
||||
).fetchall()
|
||||
session.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.gpu_queue.recover_orphaned_gpu_jobs")
|
||||
def recover_orphaned_gpu_jobs() -> int:
|
||||
"""Reset expired GPU-job leases back to pending — recovers work orphaned by an
|
||||
agent that died mid-job (no graceful release) — and convert poison-loopers
|
||||
(release/expiry cycles that never reach fail()'s attempt cap) to 'error'.
|
||||
Statements are shared with GpuJobService.recover_orphaned so the sweep and
|
||||
the service can't drift. Short beat cadence so orphans get picked back up
|
||||
quickly + the queue counts read honestly. Returns the number recovered."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from ..services.ml.gpu_jobs import recover_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
counts = {
|
||||
name: session.execute(stmt).rowcount or 0
|
||||
for name, stmt in recover_statements(datetime.now(UTC)).items()
|
||||
}
|
||||
session.commit()
|
||||
if counts["poison_expired"] or counts["poison_pending"]:
|
||||
log.warning(
|
||||
"gpu jobs poisoned -> error: %d crash-loop (expired lease), "
|
||||
"%d never-complete (pending)",
|
||||
counts["poison_expired"], counts["poison_pending"],
|
||||
)
|
||||
return counts["recovered"]
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.gpu_queue.reprocess_gpu_jobs")
|
||||
def reprocess_gpu_jobs(task_name: str = "ccip") -> int:
|
||||
"""Reset every done/error job of `task_name` back to pending so the agent
|
||||
re-runs the WHOLE library under the CURRENT pipeline — e.g. after adding crop
|
||||
detectors (#1202), re-cropping existing images. Heavy + operator-triggered;
|
||||
the back-catalogue won't otherwise re-process (the backfills skip images that
|
||||
already have current-version regions). Returns the number reset."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import update
|
||||
|
||||
from ..models import GpuJob
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
now = datetime.now(UTC)
|
||||
res = session.execute(
|
||||
update(GpuJob)
|
||||
.where(
|
||||
GpuJob.task == task_name,
|
||||
GpuJob.status.in_(["done", "error"]),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, lease_token=None, leased_at=None,
|
||||
lease_expires_at=None, updated_at=now,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return res.rowcount or 0
|
||||
@@ -228,15 +228,17 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
||||
# Enqueue thumbnail + ML for newly imported AND superseded images
|
||||
# (a superseded row has cleared ML + no thumbnail).
|
||||
if result.status in ("imported", "superseded"):
|
||||
from .ml import tag_and_embed
|
||||
from .ml import cpu_embed_enabled, embed_image
|
||||
from .thumbnail import generate_thumbnail
|
||||
|
||||
do_embed = cpu_embed_enabled()
|
||||
ids = list(result.member_image_ids)
|
||||
if result.image_id is not None and result.image_id not in ids:
|
||||
ids.append(result.image_id)
|
||||
for img_id in ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
if do_embed:
|
||||
embed_image.delay(img_id)
|
||||
|
||||
# If this was the last task in the batch, mark the batch complete.
|
||||
remaining = session.execute(
|
||||
|
||||
@@ -121,7 +121,7 @@ IMPORT_BATCH_KEEP_DAYS = 30
|
||||
# task.time_limit + a small buffer. task_name overrides take precedence
|
||||
# over queue overrides.
|
||||
#
|
||||
# ml queue: tag_and_embed video branch (≈20 GPU ops); time_limit=1200.
|
||||
# ml queue: embed_image video branch (≈20 GPU ops); time_limit=1200.
|
||||
# import_archive_file: shares the 'import' queue with the fast
|
||||
# single-file import_media_file, so it needs a task-name override
|
||||
# (the import queue itself stays at the 5-min default for single
|
||||
@@ -139,10 +139,9 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"download": 30,
|
||||
# Audit 2026-06-02 — maintenance/scan queues run tasks that
|
||||
# legitimately exceed the 5-min default (verify_integrity at 70m
|
||||
# hard, scan_directory at 70m hard, apply_allowlist_tags /
|
||||
# recompute_centroids / backfill_phash at 35m hard). 75 min lives
|
||||
# above the longest of those and the per-task overrides below
|
||||
# cover the outliers (backups, library audit).
|
||||
# hard, scan_directory at 70m hard, backfill_phash at 35m hard).
|
||||
# 75 min lives above the longest of those and the per-task
|
||||
# overrides below cover the outliers (backups, library audit).
|
||||
"maintenance": 75,
|
||||
"scan": 75,
|
||||
}
|
||||
|
||||
+51
-166
@@ -1,8 +1,15 @@
|
||||
"""ML Celery tasks: per-image embedding, backfill discovery, head training,
|
||||
model self-heal.
|
||||
|
||||
All run on the ml-worker (queue 'ml'). Sync sessions (Celery workers are sync
|
||||
processes), same pattern as FC-2a tasks.
|
||||
All run on the ml-worker (queue 'ml'), which under B3 (2026-07-02) is an
|
||||
OPTIONAL container: its only processing role is the CPU whole-image embed
|
||||
fallback (gated by ml_settings.cpu_embed_enabled) for stacks without a GPU
|
||||
agent — plus head training / auto-apply, which need sklearn/numpy and so
|
||||
live on this image. GPU-queue coordination (backfill enqueues, orphan
|
||||
recovery, reprocess) deliberately does NOT live here — see tasks/gpu_queue.py
|
||||
(maintenance lane), so the agent pipeline works with no ml-worker at all.
|
||||
Sync sessions (Celery workers are sync processes), same pattern as FC-2a
|
||||
tasks.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -26,8 +33,24 @@ def _is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
def cpu_embed_enabled() -> bool:
|
||||
"""Dispatch gate for the CPU embed fallback (B3, operator 2026-07-02):
|
||||
stacks that run a GPU agent and DROP the (optional) ml-worker container
|
||||
turn ml_settings.cpu_embed_enabled off, so the import hooks stop queueing
|
||||
embed work into a queue nothing consumes — the daily GPU 'embed' backfill
|
||||
covers those images instead. Opens its own short session because the four
|
||||
dispatch sites sit in different session scopes; defaults ON when the
|
||||
settings row is missing (a fresh install must work agent-less)."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
val = session.execute(
|
||||
select(MLSettings.cpu_embed_enabled).where(MLSettings.id == 1)
|
||||
).scalar_one_or_none()
|
||||
return True if val is None else bool(val)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.tag_and_embed",
|
||||
name="backend.app.tasks.ml.embed_image",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||
retry_backoff=5,
|
||||
@@ -44,13 +67,21 @@ def _is_video(path: Path) -> bool:
|
||||
soft_time_limit=900, # 15 min
|
||||
time_limit=1200, # 20 min hard
|
||||
)
|
||||
def tag_and_embed(self, image_id: int) -> dict:
|
||||
"""Compute + store one image's SigLIP embedding.
|
||||
def embed_image(self, image_id: int) -> dict:
|
||||
"""Compute + store one image's whole-image SigLIP embedding — the CPU
|
||||
fallback path (B3, operator 2026-07-02): this is the ml-worker's ONLY
|
||||
processing role, keeping search/similarity/head-suggestions alive on
|
||||
deployments without a GPU agent. Detection, cropping and CCIP are
|
||||
deliberately agent-only, and their backfill predicates read image_region /
|
||||
gpu_job state — never image_record.siglip_embedding — so a CPU whole-image
|
||||
embed can NEVER mark crop work as done. (Renamed from tag_and_embed —
|
||||
Camie tagging was retired #1189; the old name kept implying a tagging step
|
||||
that no longer exists.)
|
||||
|
||||
Video (#747): sample frames at a fixed cadence (ml_settings
|
||||
video_frame_interval_seconds, capped at video_max_frames) and mean-pool the
|
||||
per-frame SigLIP embeddings. On no-frames returns status='no_frames' (not an
|
||||
error). (Camie tagging was retired #1189 — heads + CCIP are the tag source.)
|
||||
per-frame SigLIP embeddings — the same shape as the GPU agent's video
|
||||
handling. On no-frames returns status='no_frames' (not an error).
|
||||
"""
|
||||
import time
|
||||
|
||||
@@ -84,9 +115,9 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
f"image_id={image_id} path={record.path} mime={record.mime} "
|
||||
f"bytes={record.size_bytes} video={is_vid}"
|
||||
)
|
||||
log.info("tag_and_embed start: %s", ctx)
|
||||
log.info("embed_image start: %s", ctx)
|
||||
if not src.is_file():
|
||||
log.warning("tag_and_embed file missing on disk: %s", ctx)
|
||||
log.warning("embed_image file missing on disk: %s", ctx)
|
||||
return {"status": "file_missing", "image_id": image_id}
|
||||
|
||||
phase = "load_models"
|
||||
@@ -102,7 +133,7 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
vprobe = safe_probe.probe_video(src)
|
||||
if not vprobe.ok:
|
||||
log.warning(
|
||||
"tag_and_embed bad video (%s): %s", vprobe.reason, ctx
|
||||
"embed_image bad video (%s): %s", vprobe.reason, ctx
|
||||
)
|
||||
return {
|
||||
"status": "bad_video", "image_id": image_id,
|
||||
@@ -130,7 +161,7 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
t0 = time.monotonic()
|
||||
embedding = embedder.infer(src)
|
||||
log.info(
|
||||
"tag_and_embed embedded in %.1fs: %s",
|
||||
"embed_image embedded in %.1fs: %s",
|
||||
time.monotonic() - t0, ctx,
|
||||
)
|
||||
|
||||
@@ -141,7 +172,7 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
session.commit()
|
||||
except SoftTimeLimitExceeded:
|
||||
log.error(
|
||||
"tag_and_embed TIMED OUT after %.0fs in phase=%s: %s",
|
||||
"embed_image TIMED OUT after %.0fs in phase=%s: %s",
|
||||
_elapsed(), phase, ctx,
|
||||
)
|
||||
# Re-raise as SoftTimeLimitExceeded (preserves the 'timeout' status in
|
||||
@@ -155,12 +186,12 @@ def tag_and_embed(self, image_id: int) -> dict:
|
||||
# ORIGINAL so the type is preserved; just make sure it's logged with
|
||||
# context first.
|
||||
log.exception(
|
||||
"tag_and_embed FAILED in phase=%s after %.0fs: %s",
|
||||
"embed_image FAILED in phase=%s after %.0fs: %s",
|
||||
phase, _elapsed(), ctx,
|
||||
)
|
||||
raise
|
||||
|
||||
log.info("tag_and_embed ok in %.1fs: %s", _elapsed(), ctx)
|
||||
log.info("embed_image ok in %.1fs: %s", _elapsed(), ctx)
|
||||
return {"status": "ok", "image_id": image_id}
|
||||
|
||||
|
||||
@@ -222,13 +253,17 @@ def _sample_video_frames(
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.backfill", bind=True)
|
||||
def backfill(self) -> int:
|
||||
"""Enqueue tag_and_embed (embed-only) for images with no SigLIP embedding.
|
||||
"""Enqueue embed_image (embed-only) for images with no SigLIP embedding.
|
||||
Keyset pagination by id ASC (restart-safe).
|
||||
|
||||
NB: a siglip MODEL-VERSION mismatch (an operator model swap, #1190) is NOT
|
||||
re-embedded here — the CPU ml-worker can't churn the library at 384/512px;
|
||||
the GPU agent owns version re-embeds via the 'embed' job.
|
||||
"""
|
||||
if not cpu_embed_enabled():
|
||||
log.info("cpu backfill skipped: cpu_embed_enabled is off (B3 — the "
|
||||
"GPU 'embed' backfill owns whole-image embeds on this stack)")
|
||||
return 0
|
||||
SessionLocal = _sync_session_factory()
|
||||
enqueued = 0
|
||||
last_id = 0
|
||||
@@ -244,7 +279,7 @@ def backfill(self) -> int:
|
||||
if not rows:
|
||||
break
|
||||
for image_id in rows:
|
||||
tag_and_embed.delay(image_id)
|
||||
embed_image.delay(image_id)
|
||||
enqueued += 1
|
||||
last_id = rows[-1]
|
||||
return enqueued
|
||||
@@ -405,156 +440,6 @@ def scheduled_apply_head_tags() -> str:
|
||||
return "dispatched"
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.enqueue_gpu_backfill")
|
||||
def enqueue_gpu_backfill(task_name: str) -> int:
|
||||
"""Enqueue a gpu_job for every image that still needs `task_name` (one
|
||||
INSERT…SELECT, so it scales to a full library). The desktop agent drains the
|
||||
queue over HTTP. Returns the number enqueued.
|
||||
|
||||
'siglip' gates on the RESULT (no concept region yet) rather than on a prior
|
||||
job, so it picks up the back-catalogue of images that were CCIP-embedded
|
||||
before concept crops existed — without re-touching their figure/CCIP regions.
|
||||
|
||||
An ERRORED job is a tombstone for its (image, task): no variant re-enqueues
|
||||
it. Retry is deliberate-only (/retry_errors), which also means an errored
|
||||
back-catalogue needs one "Retry errored jobs" press after a model swap.
|
||||
Before the tombstone rule, this loop re-minted a fresh doomed job for every
|
||||
permanently-bad file each run — ~24 duplicate error rows/day per file (the
|
||||
2026-07-02 "unprocessable" flood)."""
|
||||
from sqlalchemy import exists, insert, literal, or_
|
||||
from sqlalchemy import select as sa_select
|
||||
|
||||
from ..models import GpuJob, ImageRecord, ImageRegion, MLSettings
|
||||
from ..services.ml.gpu_jobs import error_dedupe_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
# Prune stale tombstones first (loop-era duplicates + rows made moot by
|
||||
# a later success), so 'error' reads as one row per distinct failing
|
||||
# file and the skip-guards below see a clean picture.
|
||||
pruned = sum(
|
||||
session.execute(s).rowcount or 0 for s in error_dedupe_statements()
|
||||
)
|
||||
if pruned:
|
||||
log.info("gpu backfill: pruned %d stale/duplicate error rows", pruned)
|
||||
cur_version = session.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
).scalar_one()
|
||||
if task_name == "embed":
|
||||
# Whole-image GPU re-embed (#1190): images with no embedding, or one
|
||||
# stamped under a DIFFERENT model version (an operator model swap).
|
||||
stale = or_(
|
||||
ImageRecord.siglip_embedding.is_(None),
|
||||
ImageRecord.siglip_model_version.is_(None),
|
||||
ImageRecord.siglip_model_version != cur_version,
|
||||
)
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "embed",
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("embed"), literal("pending")
|
||||
).where(stale).where(~blocked)
|
||||
elif task_name == "siglip":
|
||||
# Concept-crop re-embed: enqueue when there's no concept region AT THE
|
||||
# CURRENT model version — so a model swap re-triggers crops too, not
|
||||
# only the never-embedded back-catalogue.
|
||||
has_current_concept = exists().where(
|
||||
ImageRegion.image_record_id == ImageRecord.id,
|
||||
ImageRegion.kind == "concept",
|
||||
ImageRegion.embedding_version == cur_version,
|
||||
)
|
||||
# 'error' blocks too — tombstone rule, see docstring.
|
||||
blocked = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == "siglip",
|
||||
GpuJob.status.in_(["pending", "leased", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal("siglip"), literal("pending")
|
||||
).where(~has_current_concept).where(~blocked)
|
||||
else:
|
||||
# ANY prior row blocks — including 'error' (tombstone rule, see
|
||||
# docstring): pre-fix this branch ran HOURLY and was the loop.
|
||||
already = exists().where(
|
||||
GpuJob.image_record_id == ImageRecord.id,
|
||||
GpuJob.task == task_name,
|
||||
GpuJob.status.in_(["pending", "leased", "done", "error"]),
|
||||
)
|
||||
sel = sa_select(
|
||||
ImageRecord.id, literal(task_name), literal("pending")
|
||||
).where(~already)
|
||||
# RETURNING + count: result.rowcount is unreliable for INSERT…SELECT.
|
||||
rows = session.execute(
|
||||
insert(GpuJob)
|
||||
.from_select(["image_record_id", "task", "status"], sel)
|
||||
.returning(GpuJob.id)
|
||||
).fetchall()
|
||||
session.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.recover_orphaned_gpu_jobs")
|
||||
def recover_orphaned_gpu_jobs() -> int:
|
||||
"""Reset expired GPU-job leases back to pending — recovers work orphaned by an
|
||||
agent that died mid-job (no graceful release) — and convert poison-loopers
|
||||
(release/expiry cycles that never reach fail()'s attempt cap) to 'error'.
|
||||
Statements are shared with GpuJobService.recover_orphaned so the sweep and
|
||||
the service can't drift. Short beat cadence so orphans get picked back up
|
||||
quickly + the queue counts read honestly. Returns the number recovered."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from ..services.ml.gpu_jobs import recover_statements
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
counts = {
|
||||
name: session.execute(stmt).rowcount or 0
|
||||
for name, stmt in recover_statements(datetime.now(UTC)).items()
|
||||
}
|
||||
session.commit()
|
||||
if counts["poison_expired"] or counts["poison_pending"]:
|
||||
log.warning(
|
||||
"gpu jobs poisoned -> error: %d crash-loop (expired lease), "
|
||||
"%d never-complete (pending)",
|
||||
counts["poison_expired"], counts["poison_pending"],
|
||||
)
|
||||
return counts["recovered"]
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.reprocess_gpu_jobs")
|
||||
def reprocess_gpu_jobs(task_name: str = "ccip") -> int:
|
||||
"""Reset every done/error job of `task_name` back to pending so the agent
|
||||
re-runs the WHOLE library under the CURRENT pipeline — e.g. after adding crop
|
||||
detectors (#1202), re-cropping existing images. Heavy + operator-triggered;
|
||||
the back-catalogue won't otherwise re-process (the backfills skip images that
|
||||
already have current-version regions). Returns the number reset."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import update
|
||||
|
||||
from ..models import GpuJob
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
now = datetime.now(UTC)
|
||||
res = session.execute(
|
||||
update(GpuJob)
|
||||
.where(
|
||||
GpuJob.task == task_name,
|
||||
GpuJob.status.in_(["done", "error"]),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, lease_token=None, leased_at=None,
|
||||
lease_expires_at=None, updated_at=now,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
<template>
|
||||
<MaintenanceTile
|
||||
icon="mdi-nuke"
|
||||
title="Reset content tagging (whole instance)"
|
||||
blurb="Delete ALL general/character tags and their applications — a start-over. Requires a confirmation code."
|
||||
destructive
|
||||
>
|
||||
<p class="text-body-2 mb-2">
|
||||
Deletes every <code>general</code> and <code>character</code> tag and
|
||||
removes them from every image — <strong>including the examples the
|
||||
tagging heads learned from</strong>. Suggestions will <strong>not</strong>
|
||||
repopulate on their own: you re-tag from scratch, and the heads retrain
|
||||
from your new tags as they accumulate. Fandoms and series (with their
|
||||
page order) are kept.
|
||||
</p>
|
||||
<v-alert type="error" variant="tonal" density="compact" class="mb-3">
|
||||
Irreversible — no undo except restoring a DB backup
|
||||
(Settings → Maintenance → Backup). Back one up first.
|
||||
</v-alert>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingPreview"
|
||||
class="mb-3"
|
||||
@click="onPreview"
|
||||
>Preview content-tag reset</v-btn>
|
||||
|
||||
<div v-if="preview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ preview.count }}</strong> content tag(s)
|
||||
<span v-for="(n, k) in preview.by_kind" :key="k" class="fc-muted">
|
||||
({{ k }}: {{ n }})
|
||||
</span>
|
||||
across <strong>{{ preview.applications }}</strong> image
|
||||
application(s).
|
||||
</p>
|
||||
<SampleNameGrid
|
||||
v-if="preview.sample_names?.length"
|
||||
:names="preview.sample_names" class="mb-3"
|
||||
/>
|
||||
<template v-if="preview.count">
|
||||
<p class="text-body-2 mb-2">
|
||||
To arm the reset, type the confirmation code
|
||||
<code class="fc-code">{{ preview.confirm }}</code> below.
|
||||
</p>
|
||||
<div class="d-flex align-center mb-1" style="gap: 12px">
|
||||
<v-text-field
|
||||
v-model="typed" density="compact" hide-details variant="outlined"
|
||||
label="Confirmation code" style="max-width: 200px"
|
||||
autocomplete="off" spellcheck="false"
|
||||
/>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete-alert"
|
||||
:disabled="typed !== preview.confirm"
|
||||
:loading="committing"
|
||||
@click="onCommit"
|
||||
>Delete {{ preview.count }} tag(s) +
|
||||
{{ preview.applications }} application(s)</v-btn>
|
||||
</div>
|
||||
<p class="fc-muted text-caption mb-0">
|
||||
The code is derived from the counts above — if tagging changes
|
||||
between preview and apply, the server rejects the stale code.
|
||||
</p>
|
||||
</template>
|
||||
</div>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import SampleNameGrid from '../common/SampleNameGrid.vue'
|
||||
import { useAdminStore } from '../../stores/admin.js'
|
||||
|
||||
const store = useAdminStore()
|
||||
const preview = ref(null)
|
||||
const loadingPreview = ref(false)
|
||||
const committing = ref(false)
|
||||
const typed = ref('')
|
||||
|
||||
async function onPreview() {
|
||||
loadingPreview.value = true
|
||||
typed.value = ''
|
||||
try {
|
||||
preview.value = await store.resetContentTagging({ dryRun: true })
|
||||
} catch (e) {
|
||||
toast({ text: `Preview failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
loadingPreview.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCommit() {
|
||||
committing.value = true
|
||||
try {
|
||||
const res = await store.resetContentTagging({
|
||||
dryRun: false, confirm: typed.value,
|
||||
})
|
||||
toast({ text: `Deleted ${res.deleted} content tag(s) — re-tagging starts fresh`, type: 'success' })
|
||||
preview.value = null
|
||||
typed.value = ''
|
||||
} catch (e) {
|
||||
toast({ text: `Reset rejected: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
committing.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-code {
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
border-radius: 4px; padding: 2px 8px;
|
||||
font-family: 'JetBrains Mono', monospace; font-weight: 700;
|
||||
letter-spacing: 0.06em;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,77 @@
|
||||
<template>
|
||||
<v-card class="mb-4">
|
||||
<CardHeading icon="mdi-download" title="Downloads (last 24h)">
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" size="small" rounded="pill"
|
||||
to="/subscriptions?tab=downloads"
|
||||
>
|
||||
Open subscriptions
|
||||
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||
</v-btn>
|
||||
</CardHeading>
|
||||
<v-card-text>
|
||||
<div class="d-flex align-center flex-wrap mb-2" style="gap: 8px">
|
||||
<v-chip size="small" variant="tonal" color="success">
|
||||
{{ stats.ok }} ok
|
||||
</v-chip>
|
||||
<v-chip
|
||||
size="small" variant="tonal"
|
||||
:color="stats.error ? 'error' : undefined"
|
||||
>
|
||||
{{ stats.error }} failed
|
||||
</v-chip>
|
||||
<v-chip v-if="stats.running" size="small" variant="tonal" color="accent">
|
||||
{{ stats.running }} running
|
||||
</v-chip>
|
||||
<v-chip v-if="stats.pending" size="small" variant="tonal">
|
||||
{{ stats.pending }} pending
|
||||
</v-chip>
|
||||
<v-chip v-if="stats.skipped" size="small" variant="tonal">
|
||||
{{ stats.skipped }} skipped
|
||||
</v-chip>
|
||||
</div>
|
||||
<p v-if="!failing.length" class="fc-muted text-body-2 mb-0">
|
||||
All subscription sources healthy.
|
||||
</p>
|
||||
<p v-else class="text-body-2 mb-0">
|
||||
<b class="fc-bad">{{ failing.length }}</b> failing source(s):
|
||||
<span class="fc-muted">{{ failingNames }}</span>
|
||||
</p>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted } from 'vue'
|
||||
import { storeToRefs } from 'pinia'
|
||||
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||
|
||||
const store = useDownloadsStore()
|
||||
const { stats, failing } = storeToRefs(store)
|
||||
let pollId = null
|
||||
|
||||
const failingNames = computed(() => {
|
||||
const names = failing.value.map((s) => s.artist_name || s.url).slice(0, 3)
|
||||
const extra = failing.value.length - names.length
|
||||
return names.join(', ') + (extra > 0 ? ` +${extra} more` : '')
|
||||
})
|
||||
|
||||
function poll() {
|
||||
store.loadStats(24)
|
||||
store.loadFailing()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
poll()
|
||||
pollId = setInterval(() => { if (!document.hidden) poll() }, 30000)
|
||||
})
|
||||
onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-bad { color: rgb(var(--v-theme-error)); }
|
||||
</style>
|
||||
@@ -0,0 +1,110 @@
|
||||
<template>
|
||||
<v-card class="mb-4">
|
||||
<CardHeading icon="mdi-expansion-card" title="GPU agent pipeline">
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" size="small" rounded="pill"
|
||||
@click="$emit('open-maintenance')"
|
||||
>
|
||||
Open maintenance
|
||||
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||
</v-btn>
|
||||
</CardHeading>
|
||||
<v-card-text>
|
||||
<div class="fc-cells mb-2">
|
||||
<div class="fc-cell">
|
||||
<div class="fc-cell__n">{{ q.pending }}</div>
|
||||
<div class="fc-cell__l">pending</div>
|
||||
</div>
|
||||
<div class="fc-cell">
|
||||
<div class="fc-cell__n">{{ q.leased }}</div>
|
||||
<div class="fc-cell__l">in flight</div>
|
||||
</div>
|
||||
<div class="fc-cell">
|
||||
<div class="fc-cell__n fc-good">{{ q.done }}</div>
|
||||
<div class="fc-cell__l">done</div>
|
||||
</div>
|
||||
<div class="fc-cell">
|
||||
<div class="fc-cell__n" :class="q.error ? 'fc-bad' : ''">{{ q.error }}</div>
|
||||
<div class="fc-cell__l">errored</div>
|
||||
</div>
|
||||
</div>
|
||||
<p v-if="!q.error" class="fc-muted text-body-2 mb-0">
|
||||
No failed jobs — the pipeline is clean. Work drains whenever the
|
||||
desktop agent is running.
|
||||
</p>
|
||||
<p v-else class="text-body-2 mb-0">
|
||||
Triage: <b>{{ triage.defect }}</b> defective file(s) ·
|
||||
{{ triage.file_ok }} file-ok · {{ triage.unclassified }} unprobed
|
||||
<span v-if="reasonSummary" class="fc-muted">— {{ reasonSummary }}</span>
|
||||
<br>
|
||||
<span class="fc-muted text-caption">
|
||||
Recover defective files from Maintenance → Failed processing.
|
||||
</span>
|
||||
</p>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
import { useGpuStore } from '../../stores/gpu.js'
|
||||
|
||||
defineEmits(['open-maintenance'])
|
||||
|
||||
const store = useGpuStore()
|
||||
const q = ref({ pending: 0, leased: 0, done: 0, error: 0 })
|
||||
const triage = ref({ defect: 0, file_ok: 0, unclassified: 0 })
|
||||
const byClass = ref({})
|
||||
let pollId = null
|
||||
let lastErrorCount = -1
|
||||
|
||||
const reasonSummary = computed(() =>
|
||||
Object.entries(byClass.value)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 3)
|
||||
.map(([k, n]) => `${k.replaceAll('_', ' ')} ${n}`)
|
||||
.join(' · '))
|
||||
|
||||
async function poll() {
|
||||
try {
|
||||
q.value = await store.status()
|
||||
// The triage detail is only worth a second call when the error count
|
||||
// actually moved (it's a 500-row join server-side).
|
||||
if (q.value.error !== lastErrorCount) {
|
||||
lastErrorCount = q.value.error
|
||||
if (q.value.error > 0) {
|
||||
const body = await store.errors()
|
||||
triage.value = body.triage
|
||||
byClass.value = body.by_class
|
||||
} else {
|
||||
triage.value = { defect: 0, file_ok: 0, unclassified: 0 }
|
||||
byClass.value = {}
|
||||
}
|
||||
}
|
||||
} catch { /* non-fatal — panel just shows the last snapshot */ }
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
poll()
|
||||
pollId = setInterval(() => { if (!document.hidden) poll() }, 5000)
|
||||
})
|
||||
onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-cells { display: flex; gap: 28px; }
|
||||
.fc-cell__n {
|
||||
font-size: 20px; font-weight: 700; line-height: 1.1;
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
}
|
||||
.fc-cell__l {
|
||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||
.fc-bad { color: rgb(var(--v-theme-error)); }
|
||||
</style>
|
||||
@@ -2,7 +2,7 @@
|
||||
<MaintenanceTile
|
||||
icon="mdi-brain"
|
||||
title="Concept heads (the learning suggester)"
|
||||
blurb="Train the per-concept heads that turn your tags into suggestions — they replace Camie and sharpen every time you accept or reject."
|
||||
blurb="Train the per-concept heads that turn your tags into suggestions — they learn from your library and sharpen every time you accept or reject."
|
||||
:open="running"
|
||||
>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
|
||||
@@ -90,10 +90,14 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { reactive, watch } from 'vue'
|
||||
import { onMounted, reactive, watch } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
// Self-sufficient since the Import tab dissolved (2026-07-02): this form now
|
||||
// lives in Maintenance → Ingestion & filters and loads its own settings
|
||||
// instead of relying on the old tab's mount hook.
|
||||
onMounted(() => { if (!store.settings) store.loadSettings() })
|
||||
// Labelled stops so the less-initiated get the gist without knowing what a
|
||||
// Hamming distance is. 0 = byte-for-byte only; 10 = the shipped default.
|
||||
const PHASH_TICKS = { 0: 'Exact', 4: 'Strict', 10: 'Default', 16: 'Loose' }
|
||||
|
||||
@@ -1,254 +0,0 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<CardHeading title="Recent import tasks">
|
||||
<v-spacer />
|
||||
<v-select
|
||||
v-model="statusFilter" :items="statusOptions" density="compact"
|
||||
hide-details style="max-width: 180px;" @update:model-value="onFilterChange"
|
||||
/>
|
||||
<v-btn variant="text" rounded="pill" size="small" @click="onRefresh">
|
||||
<v-icon start>mdi-refresh</v-icon>
|
||||
Refresh
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:disabled="!hasFailed" @click="onRetryFailed"
|
||||
>
|
||||
Retry failed
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:disabled="!hasStuck" @click="onClearStuckOpen"
|
||||
>
|
||||
Clear stuck…
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="error"
|
||||
@click="onClearOpen"
|
||||
>
|
||||
Clear completed…
|
||||
</v-btn>
|
||||
</CardHeading>
|
||||
<v-data-table-virtual
|
||||
:headers="headers" :items="store.tasks" :loading="store.tasksLoading"
|
||||
height="480" density="compact" fixed-header no-data-text="No tasks yet — trigger a scan above."
|
||||
>
|
||||
<template #item.status="{ item }">
|
||||
<v-chip :color="statusColor(item.status)" size="small" variant="tonal">
|
||||
{{ item.status }}
|
||||
</v-chip>
|
||||
</template>
|
||||
<template #item.source_path="{ item }">
|
||||
<span :title="item.source_path">{{ shorten(item.source_path) }}</span>
|
||||
</template>
|
||||
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
||||
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
||||
<template #item.error="{ item }">
|
||||
<button
|
||||
v-if="item.error" type="button" class="fc-err-link text-caption"
|
||||
@click="openError(`Task ${item.id} failed`, item.error)"
|
||||
title="Click for full error"
|
||||
>{{ shorten(item.error, 60) }}</button>
|
||||
</template>
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
v-if="item.status === 'failed'"
|
||||
icon size="x-small" variant="text"
|
||||
:loading="refetching === item.id"
|
||||
@click="onRefetch(item)"
|
||||
>
|
||||
<v-icon size="small">mdi-cloud-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">
|
||||
Re-fetch original (re-download from source)
|
||||
</v-tooltip>
|
||||
</v-btn>
|
||||
</template>
|
||||
</v-data-table-virtual>
|
||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
||||
</div>
|
||||
|
||||
<v-dialog v-model="clearDialog" max-width="400">
|
||||
<v-card>
|
||||
<v-card-title>Clear completed tasks</v-card-title>
|
||||
<v-card-text>
|
||||
<v-select
|
||||
v-model="clearAgeDays" label="Older than"
|
||||
:items="[
|
||||
{ title: 'All finished', value: 0 },
|
||||
{ title: '1 day', value: 1 },
|
||||
{ title: '7 days', value: 7 },
|
||||
{ title: '30 days', value: 30 }
|
||||
]"
|
||||
/>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn @click="clearDialog = false">Cancel</v-btn>
|
||||
<v-btn color="error" rounded="pill" @click="onClearConfirm">Clear</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<v-dialog v-model="clearStuckDialog" max-width="480">
|
||||
<v-card>
|
||||
<v-card-title>Clear stuck tasks</v-card-title>
|
||||
<v-card-text>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
Force every <strong>pending / queued / processing</strong> task to
|
||||
<strong>failed</strong> and finalize any active batch that
|
||||
has no remaining work. Use this when the automatic recovery
|
||||
sweep keeps re-queueing the same row (e.g., corrupt file in
|
||||
an autoretry loop, or worker model missing).
|
||||
</v-alert>
|
||||
<p class="text-body-2">
|
||||
Tasks remain in the database with status=<code>failed</code>;
|
||||
click <em>Retry failed</em> once the underlying cause is
|
||||
resolved to re-queue them.
|
||||
</p>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn @click="clearStuckDialog = false">Cancel</v-btn>
|
||||
<v-btn color="warning" rounded="pill" @click="onClearStuckConfirm">Clear stuck</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<ErrorDetailModal
|
||||
v-model="showErrorModal"
|
||||
:title="errorModalTitle"
|
||||
:message="errorModalMessage"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import { computed, ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
|
||||
const store = useImportStore()
|
||||
const statusFilter = ref(null)
|
||||
const clearDialog = ref(false)
|
||||
// Click-to-open modal for full error text (operator-flagged 2026-05-26
|
||||
// — the prior :title="..." tooltip cramped multi-line SQLAlchemy
|
||||
// tracebacks into an unusable popup with no copy-paste affordance).
|
||||
const showErrorModal = ref(false)
|
||||
const errorModalTitle = ref('')
|
||||
const errorModalMessage = ref('')
|
||||
|
||||
function openError(title, message) {
|
||||
errorModalTitle.value = title
|
||||
errorModalMessage.value = message || ''
|
||||
showErrorModal.value = true
|
||||
}
|
||||
const clearAgeDays = ref(7)
|
||||
const clearStuckDialog = ref(false)
|
||||
|
||||
const statusOptions = [
|
||||
{ title: 'All', value: null },
|
||||
{ title: 'Pending', value: 'pending' },
|
||||
{ title: 'Queued', value: 'queued' },
|
||||
{ title: 'Processing', value: 'processing' },
|
||||
{ title: 'Complete', value: 'complete' },
|
||||
{ title: 'Skipped', value: 'skipped' },
|
||||
{ title: 'Failed', value: 'failed' }
|
||||
]
|
||||
|
||||
const headers = [
|
||||
{ title: 'Status', key: 'status', sortable: false, width: 120 },
|
||||
{ title: 'Source', key: 'source_path', sortable: false },
|
||||
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
||||
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
||||
{ title: 'Note', key: 'error', sortable: false },
|
||||
{ title: '', key: 'actions', sortable: false, width: 56 }
|
||||
]
|
||||
|
||||
const refetching = ref(null)
|
||||
const _REFETCH_MSG = {
|
||||
refetch_queued: { text: 'Re-fetch queued — re-downloading from source', type: 'success' },
|
||||
no_source: { text: 'No re-fetchable source (filesystem import — replace the file manually)', type: 'info' },
|
||||
already_refetched: { text: 'Already re-fetched once', type: 'info' },
|
||||
}
|
||||
async function onRefetch(item) {
|
||||
refetching.value = item.id
|
||||
try {
|
||||
const res = await store.refetchTask(item.id)
|
||||
const msg = _REFETCH_MSG[res.status] || { text: `Re-fetch: ${res.status}`, type: 'info' }
|
||||
toast(msg)
|
||||
} catch (e) {
|
||||
toast({ text: `Re-fetch failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
refetching.value = null
|
||||
}
|
||||
}
|
||||
|
||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||
const hasStuck = computed(() => store.tasks.some(
|
||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||
))
|
||||
|
||||
function statusColor(s) {
|
||||
return {
|
||||
complete: 'success',
|
||||
skipped: 'warning',
|
||||
failed: 'error',
|
||||
processing: 'accent',
|
||||
queued: 'info',
|
||||
pending: 'info'
|
||||
}[s] || 'default'
|
||||
}
|
||||
function shorten(s, max = 90) {
|
||||
if (!s) return ''
|
||||
if (s.length <= max) return s
|
||||
const head = Math.floor((max - 3) * 0.6)
|
||||
const tail = max - 3 - head
|
||||
return s.slice(0, head) + '...' + s.slice(-tail)
|
||||
}
|
||||
function formatBytes(b) {
|
||||
if (!b) return ''
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB']
|
||||
let i = 0; let v = b
|
||||
while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ }
|
||||
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
||||
}
|
||||
function formatDate(s) {
|
||||
try { return new Date(s).toLocaleString() } catch { return s }
|
||||
}
|
||||
|
||||
async function onRefresh() { await store.loadTasks(true) }
|
||||
function onFilterChange() { store.setStatusFilter(statusFilter.value); store.loadTasks(true) }
|
||||
async function onLoadMore() { await store.loadTasks(false) }
|
||||
async function onRetryFailed() { await store.retryFailed() }
|
||||
function onClearOpen() { clearDialog.value = true }
|
||||
async function onClearConfirm() {
|
||||
await store.clearCompleted(clearAgeDays.value)
|
||||
clearDialog.value = false
|
||||
}
|
||||
function onClearStuckOpen() { clearStuckDialog.value = true }
|
||||
async function onClearStuckConfirm() {
|
||||
await store.clearStuck()
|
||||
clearStuckDialog.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-err-link {
|
||||
/* Truncated error preview as a clickable button — opens
|
||||
ErrorDetailModal with the full text. Inherits the row's font
|
||||
sizing so it doesn't visually drift from the prior tooltip-bearing
|
||||
span. */
|
||||
color: rgb(var(--v-theme-error, 220 80 80));
|
||||
background: transparent;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
font: inherit;
|
||||
text-align: left;
|
||||
text-decoration: underline dotted;
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-err-link:hover { text-decoration: underline; }
|
||||
</style>
|
||||
@@ -1,97 +0,0 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<v-card-title>Trigger scan</v-card-title>
|
||||
<v-card-text>
|
||||
<div v-if="store.activeBatch" class="d-flex align-center mb-3" style="gap: 12px;">
|
||||
<v-progress-circular
|
||||
indeterminate color="accent" size="20"
|
||||
/>
|
||||
<span>
|
||||
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
|
||||
{{ store.activeBatch.source_path || '/import' }} —
|
||||
imported {{ store.activeBatch.imported }},
|
||||
<template v-if="store.activeBatch.scan_mode === 'deep'">
|
||||
refreshed {{ store.activeBatch.refreshed || 0 }},
|
||||
</template>
|
||||
skipped {{ store.activeBatch.skipped }},
|
||||
failed {{ store.activeBatch.failed }} /
|
||||
{{ store.activeBatch.total_files }} files
|
||||
</span>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:loading="clearing" @click="onClearStuck"
|
||||
>
|
||||
Clear stuck
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<p class="text-body-2 mb-3">
|
||||
<span v-if="!store.activeBatch">
|
||||
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
|
||||
new files only.
|
||||
<strong>Deep scan</strong> additionally re-walks already-imported
|
||||
files so updated sidecar metadata (post title/date/attribution) and
|
||||
previously-NULL phashes / artist links get refreshed. Use after
|
||||
bulk-downloading fresh sidecars for existing content. Both modes
|
||||
route non-media + sidecar pairs through PostAttachment capture.
|
||||
</span>
|
||||
<span v-else>
|
||||
An active batch is in progress. Wait for it to finish, or click
|
||||
<em>Clear stuck</em> above if it has been wedged with no
|
||||
measurable progress.
|
||||
</span>
|
||||
</p>
|
||||
|
||||
<div class="d-flex flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" rounded="pill"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'quick'"
|
||||
@click="trigger('quick')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-scan</v-icon>
|
||||
Quick scan
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="secondary" rounded="pill" variant="tonal"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'deep'"
|
||||
@click="trigger('deep')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-plus-outline</v-icon>
|
||||
Deep scan
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
|
||||
{{ store.triggerError }}
|
||||
</v-alert>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
const busy = ref(null)
|
||||
const clearing = ref(false)
|
||||
|
||||
async function trigger(mode) {
|
||||
busy.value = mode
|
||||
try { await store.triggerScan(mode) } catch {} finally { busy.value = null }
|
||||
}
|
||||
|
||||
async function onClearStuck() {
|
||||
clearing.value = true
|
||||
try {
|
||||
await store.clearStuck()
|
||||
} catch {
|
||||
// store surfaces error via triggerError if needed
|
||||
} finally {
|
||||
clearing.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -1,16 +1,33 @@
|
||||
<template>
|
||||
<MaintenanceTile
|
||||
icon="mdi-refresh"
|
||||
title="ML backfill"
|
||||
blurb="Compute SigLIP embeddings on images missing them."
|
||||
title="CPU embedding backfill"
|
||||
blurb="Whole-image embeddings without a GPU agent — the built-in fallback."
|
||||
:open="busy"
|
||||
>
|
||||
<p class="text-body-2 mb-3">
|
||||
Compute the SigLIP embedding for any image that doesn't have one yet
|
||||
(CPU). Safe to re-run. To re-embed under a NEW model, use the GPU
|
||||
agent's "Re-embed library" instead.
|
||||
Computes the whole-image SigLIP embedding for anything missing one —
|
||||
images directly, videos by sampling frames (the same approach as the
|
||||
GPU agent). Runs on the ml-worker's CPU, so search, similarity and
|
||||
head suggestions work <strong>without</strong> a GPU agent; new imports
|
||||
are embedded this way automatically. Detection, cropping and character
|
||||
(CCIP) embeddings are GPU-agent-only. Safe to re-run. To re-embed under
|
||||
a NEW model, use the GPU agent's "Re-embed library" instead.
|
||||
</p>
|
||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||
<v-switch
|
||||
v-model="enabled" color="accent" hide-details density="compact"
|
||||
:loading="saving" label="CPU embedding enabled"
|
||||
class="mb-1" @update:model-value="onToggle"
|
||||
/>
|
||||
<p class="fc-muted text-caption mb-3">
|
||||
Turn OFF if you run the GPU agent and removed the ml-worker container —
|
||||
imports then stop queueing CPU embed work nothing will consume (the
|
||||
daily GPU embed backfill covers those images instead).
|
||||
</p>
|
||||
<v-btn
|
||||
color="primary" rounded="pill" :loading="busy" :disabled="!enabled"
|
||||
@click="run"
|
||||
>
|
||||
<v-icon start>mdi-refresh</v-icon> Run backfill now
|
||||
</v-btn>
|
||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||
@@ -20,13 +37,40 @@
|
||||
|
||||
<script setup>
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import { ref } from 'vue'
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { useMLStore } from '../../stores/ml.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
const store = useMLStore()
|
||||
const busy = ref(false)
|
||||
const done = ref(false)
|
||||
const enabled = ref(true)
|
||||
const saving = ref(false)
|
||||
onMounted(async () => {
|
||||
try {
|
||||
await store.loadSettings()
|
||||
if (store.settings?.cpu_embed_enabled != null) {
|
||||
enabled.value = store.settings.cpu_embed_enabled
|
||||
}
|
||||
} catch { /* non-fatal */ }
|
||||
})
|
||||
async function onToggle() {
|
||||
saving.value = true
|
||||
try {
|
||||
await store.patchSettings({ cpu_embed_enabled: enabled.value })
|
||||
toast({
|
||||
text: enabled.value
|
||||
? 'CPU embedding on — imports queue embeds for the ml-worker'
|
||||
: 'CPU embedding off — the GPU embed backfill owns whole-image embeds',
|
||||
type: 'success',
|
||||
})
|
||||
} catch (e) {
|
||||
toast({ text: `Could not save: ${e.message}`, type: 'error' })
|
||||
enabled.value = !enabled.value
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
async function run() {
|
||||
busy.value = true
|
||||
try { await store.triggerBackfill(); done.value = true }
|
||||
@@ -34,3 +78,7 @@ async function run() {
|
||||
finally { busy.value = false }
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
|
||||
@@ -1,36 +1,59 @@
|
||||
<template>
|
||||
<div class="fc-maint">
|
||||
<p class="fc-muted text-body-2 mb-5">
|
||||
One-off backfills, tagging config and storage tools. Heads train nightly
|
||||
and auto-apply earned tags. Click a tile to open it.
|
||||
Processing, tagging and storage tools, grouped by system. Heads train
|
||||
nightly and auto-apply earned tags. Click a tile to open it.
|
||||
</p>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Backfills & reprocessing</h3>
|
||||
<p class="fc-section__hint">Re-run tagging, thumbnails, extraction and DB upkeep.</p>
|
||||
<div class="fc-tile-grid">
|
||||
<MLBackfillCard />
|
||||
<ThumbnailBackfillCard />
|
||||
<ArchiveReextractCard />
|
||||
<MissingFileRepairCard />
|
||||
<h3 class="fc-section__title">Ingestion & filters</h3>
|
||||
<p class="fc-section__hint">
|
||||
What gets imported — dedup sensitivity, size/transparency/solid-color
|
||||
filters. Applies to downloads and folder imports alike.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<ImportFiltersForm />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">GPU agent & embeddings</h3>
|
||||
<p class="fc-section__hint">
|
||||
The desktop agent that does the heavy lifting, its failure triage, and
|
||||
the CPU fallback.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<GpuAgentCard />
|
||||
<GpuTriageCard />
|
||||
<DbMaintenanceCard />
|
||||
<MLBackfillCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Tagging</h3>
|
||||
<p class="fc-section__hint">
|
||||
Suggestion thresholds, the auto-apply allowlist and tag aliases.
|
||||
Suggestion thresholds, trained heads and tag aliases.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<MLThresholdSliders />
|
||||
<HeadsCard />
|
||||
<GpuAgentCard />
|
||||
<AliasTable />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Library health</h3>
|
||||
<p class="fc-section__hint">
|
||||
Self-healing and repair: missing files, thumbnails, database upkeep.
|
||||
</p>
|
||||
<div class="fc-tile-grid">
|
||||
<MissingFileRepairCard />
|
||||
<ThumbnailBackfillCard />
|
||||
<DbMaintenanceCard />
|
||||
<ArchiveReextractCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Storage</h3>
|
||||
<p class="fc-section__hint">Database + image backups and restore.</p>
|
||||
@@ -44,6 +67,7 @@
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted } from 'vue'
|
||||
|
||||
import ImportFiltersForm from './ImportFiltersForm.vue'
|
||||
import MLBackfillCard from './MLBackfillCard.vue'
|
||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
|
||||
@@ -17,6 +17,12 @@
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<!-- The non-Celery halves of the app (2026-07-02): the GPU agent does the
|
||||
majority of processing and downloads feed the library — Activity is
|
||||
the whole-app pulse, not just the worker queues. -->
|
||||
<GpuActivityPanel @open-maintenance="$emit('open-maintenance')" />
|
||||
<DownloadsActivityPanel />
|
||||
|
||||
<!-- Recent failures pane -->
|
||||
<v-card class="mb-4">
|
||||
<CardHeading icon="mdi-alert-circle-outline" title="Recent failures (last 24h)">
|
||||
@@ -167,6 +173,10 @@ import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
import QueuesTable from './QueuesTable.vue'
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
import GpuActivityPanel from './GpuActivityPanel.vue'
|
||||
import DownloadsActivityPanel from './DownloadsActivityPanel.vue'
|
||||
|
||||
defineEmits(['open-maintenance'])
|
||||
|
||||
// Click-to-open modal for full error text. Replaces the unusable
|
||||
// :title="..." tooltip (operator-flagged 2026-05-26: SQLAlchemy
|
||||
|
||||
@@ -42,56 +42,6 @@
|
||||
</div>
|
||||
</MaintenanceTile>
|
||||
|
||||
<MaintenanceTile
|
||||
icon="mdi-tag-multiple"
|
||||
title="Reset content tagging"
|
||||
blurb="Delete all general/character tags to re-tag from scratch."
|
||||
destructive
|
||||
>
|
||||
<p class="text-body-2 mb-2">
|
||||
Deletes every <code>general</code> and <code>character</code> tag and
|
||||
removes them from every image, so you can re-tag from scratch with the
|
||||
auto-suggest. <strong>Fandoms and series (with their page order) are
|
||||
kept</strong>, and each image's saved predictions are untouched — open
|
||||
an image and its suggestions reappear.
|
||||
</p>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
Irreversible — there's no undo except restoring a DB backup.
|
||||
Back one up first (Settings → Maintenance → Backup).
|
||||
</v-alert>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingResetPreview"
|
||||
class="mb-3"
|
||||
@click="onResetPreview"
|
||||
>Preview content-tag reset</v-btn>
|
||||
|
||||
<div v-if="resetPreview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ resetPreview.count }}</strong> content tag(s)
|
||||
<span v-for="(n, k) in resetPreview.by_kind" :key="k" class="fc-muted">
|
||||
({{ k }}: {{ n }})
|
||||
</span>
|
||||
across <strong>{{ resetPreview.applications }}</strong> image
|
||||
application(s).
|
||||
</p>
|
||||
<SampleNameGrid
|
||||
v-if="resetPreview.sample_names?.length"
|
||||
:names="resetPreview.sample_names" class="mb-3"
|
||||
/>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete-alert"
|
||||
:disabled="!resetPreview.count"
|
||||
:loading="resetCommitting"
|
||||
@click="onResetCommit"
|
||||
>Delete {{ resetPreview.count }} content tag(s) +
|
||||
{{ resetPreview.applications }} application(s)</v-btn>
|
||||
</div>
|
||||
</MaintenanceTile>
|
||||
|
||||
<MaintenanceTile
|
||||
icon="mdi-format-letter-case"
|
||||
title="Standardize tag casing"
|
||||
@@ -169,16 +119,6 @@ const {
|
||||
emptyPreview: (r) => ({ count: 0, sample_names: r.sample_names || [] }),
|
||||
})
|
||||
|
||||
// Reset content tagging (general + character).
|
||||
const {
|
||||
previewData: resetPreview, previewing: loadingResetPreview,
|
||||
committing: resetCommitting, runPreview: onResetPreview, runCommit: onResetCommit,
|
||||
} = usePreviewCommit({
|
||||
preview: () => store.resetContentTagging({ dryRun: true }),
|
||||
commit: () => store.resetContentTagging({ dryRun: false }),
|
||||
emptyPreview: { count: 0, by_kind: {}, applications: 0, sample_names: [] },
|
||||
})
|
||||
|
||||
// Standardize casing. The apply DISPATCHES a self-resuming background task (no
|
||||
// poll-until-done — that would falsely report complete after the first chunk),
|
||||
// so there's no emptyPreview: leave the projection up; a truthy normResult means
|
||||
|
||||
@@ -52,14 +52,9 @@
|
||||
{{ String(store.error) }}
|
||||
</v-alert>
|
||||
|
||||
<FailingSourcesCard
|
||||
:sources="store.failing"
|
||||
:retrying-ids="sourcesStore.checkingIds"
|
||||
:retrying-all="retryingAll"
|
||||
@retry="onRetrySource"
|
||||
@retry-all="onRetryAll"
|
||||
@view-logs="onViewFailingLogs"
|
||||
/>
|
||||
<!-- The failing-sources rollup moved to the Subscriptions landing tab
|
||||
(needs-attention strip, 2026-07-02); the maintenance menu above keeps
|
||||
its bulk-retry via the same shared store action. -->
|
||||
|
||||
<div v-if="store.loading && store.events.length === 0" class="fc-dl__loading">
|
||||
<v-progress-circular indeterminate color="accent" size="36" />
|
||||
@@ -124,21 +119,17 @@ import { computed, onMounted, onUnmounted, reactive, ref, watch } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
|
||||
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import DownloadEventRow from '../downloads/DownloadEventRow.vue'
|
||||
import DownloadDetailModal from '../downloads/DownloadDetailModal.vue'
|
||||
import DownloadStatChips from './DownloadStatChips.vue'
|
||||
import DownloadActivitySparkline from './DownloadActivitySparkline.vue'
|
||||
import FailingSourcesCard from './FailingSourcesCard.vue'
|
||||
import ActiveDownloadsPanel from './ActiveDownloadsPanel.vue'
|
||||
import MaintenanceMenu from './MaintenanceMenu.vue'
|
||||
import DownloadsFilterPopover from './DownloadsFilterPopover.vue'
|
||||
|
||||
const route = useRoute()
|
||||
const store = useDownloadsStore()
|
||||
const sourcesStore = useSourcesStore()
|
||||
const filterModel = ref({ ...store.filter })
|
||||
const retryingAll = ref(false)
|
||||
|
||||
// Free-text search (client-side over the loaded events) + a toggle to
|
||||
// hide "no-change" scheduled scans (status=ok/skipped with 0 files) so
|
||||
@@ -175,51 +166,18 @@ async function refresh() {
|
||||
// rollup + stats so the operator sees it move. Passes force=true so the
|
||||
// platform cooldown is bypassed — single-source click is an explicit
|
||||
// operator override, useful for rapid auth-fix or fixture testing.
|
||||
async function onRetrySource(source) {
|
||||
try {
|
||||
await sourcesStore.checkNow(source.id, { force: true })
|
||||
toast({ text: `Retry queued for ${source.artist_name || source.platform}`, type: 'success' })
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) {
|
||||
toast({ text: 'Already running — see below', type: 'info' })
|
||||
} else {
|
||||
toast({ text: `Retry failed: ${e?.detail || e?.message || e}`, type: 'error' })
|
||||
}
|
||||
} finally {
|
||||
await Promise.all([store.loadFailing(), store.loadStats(24), store.loadFirst()])
|
||||
}
|
||||
}
|
||||
|
||||
// Bulk retry — leaves cooldown enforcement ON so N failing sources on
|
||||
// the same platform don't all retry into the rate limit the cooldown is
|
||||
// preventing. Sources deferred by cooldown will be picked up by the
|
||||
// next scan tick after the AppSetting expires. Toast tallies the three
|
||||
// outcomes so the operator can quickly read whether cooldown is the
|
||||
// dominant failure mode ("12 deferred (cooldown)" → yes, rate limit is
|
||||
// the issue).
|
||||
// Bulk retry for the maintenance menu — the shared store action keeps the
|
||||
// cooldown semantics + tally shape identical to the needs-attention card on
|
||||
// the Subscriptions tab. Toast tallies the three outcomes so the operator
|
||||
// can read whether cooldown is the dominant failure mode ("12 deferred
|
||||
// (cooldown)" → yes, rate limit is the issue).
|
||||
async function onRetryAll(sources) {
|
||||
retryingAll.value = true
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
let deferred = 0
|
||||
try {
|
||||
for (const s of sources) {
|
||||
try {
|
||||
const body = await sourcesStore.checkNow(s.id)
|
||||
if (body?.status === 'deferred') deferred += 1
|
||||
else ok += 1
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) conflict += 1
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
retryingAll.value = false
|
||||
await Promise.all([store.loadFailing(), store.loadStats(24), store.loadFirst()])
|
||||
}
|
||||
const t = await store.retryAllFailing(sources)
|
||||
await store.loadFirst()
|
||||
const parts = []
|
||||
if (ok) parts.push(`${ok} queued`)
|
||||
if (deferred) parts.push(`${deferred} deferred (cooldown)`)
|
||||
if (conflict) parts.push(`${conflict} already running`)
|
||||
if (t.ok) parts.push(`${t.ok} queued`)
|
||||
if (t.deferred) parts.push(`${t.deferred} deferred (cooldown)`)
|
||||
if (t.conflict) parts.push(`${t.conflict} already running`)
|
||||
toast({ text: parts.join(', ') || 'Nothing to retry', type: 'info' })
|
||||
}
|
||||
|
||||
@@ -276,6 +234,16 @@ onMounted(() => {
|
||||
})
|
||||
onUnmounted(stopPolling)
|
||||
|
||||
// The needs-attention card (Subscriptions tab) deep-links here with
|
||||
// ?source_id= while this tab may ALREADY be mounted (v-window keeps tabs
|
||||
// alive) — react to the query, not just the mount.
|
||||
watch(() => route.query.source_id, (v) => {
|
||||
if (v) {
|
||||
filterModel.value = { ...filterModel.value, source_id: Number(v) }
|
||||
refresh()
|
||||
}
|
||||
})
|
||||
|
||||
// Client-side date filter on the loaded page (avoids a backend round-trip
|
||||
// for the date pickers; the existing /api/downloads endpoint can grow
|
||||
// these as proper query params later if a UX need shows up).
|
||||
@@ -366,22 +334,6 @@ async function openDetail(id) {
|
||||
await store.loadOne(id)
|
||||
}
|
||||
|
||||
async function onViewFailingLogs(source) {
|
||||
// Find and open the most recent DownloadEvent for this source.
|
||||
// Reuses the existing DownloadDetailModal — same stdout/stderr/error
|
||||
// surface the row-click in the events feed shows.
|
||||
try {
|
||||
const ev = await store.loadLastForSource(source.id)
|
||||
if (!ev) {
|
||||
toast({
|
||||
text: `No download events recorded for ${source.artist_name || source.platform} yet.`,
|
||||
type: 'warning',
|
||||
})
|
||||
}
|
||||
} catch (e) {
|
||||
toast({ text: `Failed to load logs: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -18,14 +18,6 @@
|
||||
subtitle="Mark stranded pending/running events as error (also runs every 5 min)"
|
||||
@click="emit('recover-stalled')"
|
||||
/>
|
||||
<v-list-item
|
||||
:disabled="true"
|
||||
prepend-icon="mdi-download-box"
|
||||
title="Export failed logs"
|
||||
subtitle="CSV dump — v2"
|
||||
>
|
||||
<v-tooltip activator="parent" location="start">Deferred to a future release</v-tooltip>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</template>
|
||||
@@ -33,7 +25,8 @@
|
||||
<script setup>
|
||||
// The Downloads tab parent (DownloadsTab.vue) owns the actual retry/sweep
|
||||
// handlers — same toast + refresh logic already used by the failing-sources
|
||||
// RETRY ALL button. We just emit. Import-pipeline maintenance lives in
|
||||
// Settings → Imports (ImportTaskList.vue), not here.
|
||||
// RETRY ALL button. We just emit. (A permanently-disabled "Export failed
|
||||
// logs" stub sat here until 2026-07-02 — retired; the event list + detail
|
||||
// modal cover forensics.)
|
||||
const emit = defineEmits(['retry-failed', 'recover-stalled'])
|
||||
</script>
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
<template>
|
||||
<!-- Renders nothing when everything is healthy — the daily answer to
|
||||
"does anything need me?" should be silence, not an empty card. -->
|
||||
<FailingSourcesCard
|
||||
v-if="failing.length"
|
||||
class="mb-4"
|
||||
:sources="failing"
|
||||
:retrying-ids="sourcesStore.checkingIds"
|
||||
:retrying-all="retryingAll"
|
||||
@retry="onRetry"
|
||||
@retry-all="onRetryAll"
|
||||
@view-logs="onViewLogs"
|
||||
/>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { useRouter } from 'vue-router'
|
||||
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import FailingSourcesCard from './FailingSourcesCard.vue'
|
||||
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
|
||||
// The needs-attention strip on the Subscriptions LANDING tab (2026-07-02):
|
||||
// failing sources used to live below the fold of the Downloads tab, so a
|
||||
// broken subscription was invisible unless you went looking. Retry logic is
|
||||
// shared with the Downloads maintenance menu via the downloads store.
|
||||
const store = useDownloadsStore()
|
||||
const sourcesStore = useSourcesStore()
|
||||
const router = useRouter()
|
||||
const { failing } = storeToRefs(store)
|
||||
const retryingAll = ref(false)
|
||||
let pollId = null
|
||||
|
||||
async function onRetry(source) {
|
||||
try {
|
||||
const res = await store.retrySource(source)
|
||||
toast(res === 'queued'
|
||||
? { text: `Retry queued for ${source.artist_name || source.platform}`, type: 'success' }
|
||||
: { text: 'Already running — check Downloads', type: 'info' })
|
||||
} catch (e) {
|
||||
toast({ text: `Retry failed: ${e?.detail || e?.message || e}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
|
||||
async function onRetryAll(sources) {
|
||||
retryingAll.value = true
|
||||
try {
|
||||
const t = await store.retryAllFailing(sources)
|
||||
const parts = []
|
||||
if (t.ok) parts.push(`${t.ok} queued`)
|
||||
if (t.deferred) parts.push(`${t.deferred} deferred (cooldown)`)
|
||||
if (t.conflict) parts.push(`${t.conflict} already running`)
|
||||
toast({ text: parts.join(', ') || 'Nothing to retry', type: 'info' })
|
||||
} finally {
|
||||
retryingAll.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function onViewLogs(source) {
|
||||
// The Downloads tab owns the log/detail surface — deep-link into it
|
||||
// pre-filtered to this source (it watches ?source_id).
|
||||
router.push({ path: '/subscriptions', query: { tab: 'downloads', source_id: source.id } })
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
store.loadFailing()
|
||||
pollId = setInterval(() => { if (!document.hidden) store.loadFailing() }, 60000)
|
||||
})
|
||||
onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
||||
</script>
|
||||
@@ -0,0 +1,80 @@
|
||||
<template>
|
||||
<v-card v-if="arrivals.length" class="mb-4">
|
||||
<CardHeading icon="mdi-new-box" title="Recent arrivals">
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" size="small" rounded="pill"
|
||||
to="/subscriptions?tab=downloads"
|
||||
>
|
||||
All downloads
|
||||
<v-icon end size="small">mdi-arrow-right</v-icon>
|
||||
</v-btn>
|
||||
</CardHeading>
|
||||
<v-card-text class="pt-0">
|
||||
<div
|
||||
v-for="ev in arrivals" :key="ev.id"
|
||||
class="fc-arrival"
|
||||
>
|
||||
<router-link
|
||||
v-if="ev.artist_slug"
|
||||
:to="`/artist/${ev.artist_slug}`"
|
||||
class="fc-arrival__artist"
|
||||
>{{ ev.artist_name || ev.artist_slug }}</router-link>
|
||||
<span v-else class="fc-arrival__artist">{{ ev.artist_name || '—' }}</span>
|
||||
<span class="fc-arrival__meta">
|
||||
{{ ev.platform }} ·
|
||||
{{ ev.files_count ? `${ev.files_count} file(s)` : 'no new files' }}
|
||||
· {{ formatRelative(ev.started_at) }}
|
||||
</span>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
import { formatRelative } from '../../utils/date.js'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
|
||||
// "What came in?" — the other half of the landing tab's daily answer
|
||||
// (2026-07-02). Own fetch, own state: the downloads store's event list is
|
||||
// the Downloads tab's FILTERED feed; borrowing it would couple this card to
|
||||
// whatever filter the operator left that tab on.
|
||||
const api = useApi()
|
||||
const arrivals = ref([])
|
||||
let pollId = null
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const events = await api.get('/api/downloads', {
|
||||
params: { status: 'ok', limit: 25 },
|
||||
})
|
||||
// Real arrivals only — scheduled scans that found nothing are noise here.
|
||||
arrivals.value = events.filter((e) => (e.files_count || 0) > 0).slice(0, 6)
|
||||
} catch { /* non-fatal — the card just hides */ }
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
load()
|
||||
pollId = setInterval(() => { if (!document.hidden) load() }, 60000)
|
||||
})
|
||||
onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-arrival {
|
||||
display: flex; align-items: baseline; gap: 10px;
|
||||
padding: 4px 0;
|
||||
}
|
||||
.fc-arrival__artist {
|
||||
font-weight: 600; text-decoration: none;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
}
|
||||
.fc-arrival__artist:hover { text-decoration: underline; }
|
||||
.fc-arrival__meta {
|
||||
font-size: 12px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
</style>
|
||||
@@ -1,12 +1,20 @@
|
||||
<template>
|
||||
<div>
|
||||
<!-- One extension home (2026-07-02): install/manifest card (moved here
|
||||
from Settings → Overview) + the API key bar it authenticates with.
|
||||
The extension feeds THIS view — cookies + one-click sources — so its
|
||||
setup lives with the rest of the ingestion config. -->
|
||||
<h3 class="fc-section__title">Browser extension</h3>
|
||||
<p class="fc-section__hint">Install, session cookies and the API key it authenticates with.</p>
|
||||
<BrowserExtensionCard class="mb-3" />
|
||||
<ExtensionKeyBar class="mb-4" />
|
||||
|
||||
<v-alert v-if="credentialsStore.error" type="error" variant="tonal" closable class="mb-4">
|
||||
{{ String(credentialsStore.error) }}
|
||||
</v-alert>
|
||||
|
||||
<h3 class="text-h6 mb-3">Platform credentials</h3>
|
||||
<h3 class="fc-section__title mt-6">Platform credentials</h3>
|
||||
<p class="fc-section__hint">Per-platform cookies/tokens the downloader uses.</p>
|
||||
<v-row>
|
||||
<v-col
|
||||
v-for="p in platformsStore.list"
|
||||
@@ -23,7 +31,8 @@
|
||||
</v-col>
|
||||
</v-row>
|
||||
|
||||
<h3 class="text-h6 mb-3 mt-6">Downloader</h3>
|
||||
<h3 class="fc-section__title mt-6">Downloader</h3>
|
||||
<p class="fc-section__hint">Rate limits and gallery-dl behavior for source checks.</p>
|
||||
<v-card variant="outlined">
|
||||
<v-card-text v-if="importStore.settings">
|
||||
<v-row>
|
||||
@@ -52,7 +61,8 @@
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<h3 class="text-h6 mb-3 mt-6">External file-host downloads</h3>
|
||||
<h3 class="fc-section__title mt-6">External file-host downloads</h3>
|
||||
<p class="fc-section__hint">Post-linked mega/gdrive/file-host fetches.</p>
|
||||
<v-card variant="outlined">
|
||||
<v-card-text v-if="importStore.settings">
|
||||
<div class="fc-help mb-2">
|
||||
@@ -98,7 +108,8 @@
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<h3 class="text-h6 mb-3 mt-6">Schedule defaults</h3>
|
||||
<h3 class="fc-section__title mt-6">Schedule defaults</h3>
|
||||
<p class="fc-section__hint">Check cadence and failure-badge thresholds for new sources.</p>
|
||||
<v-card variant="outlined">
|
||||
<v-card-text v-if="importStore.settings">
|
||||
<v-row>
|
||||
@@ -179,6 +190,7 @@ import { onMounted, reactive, ref, watch } from 'vue'
|
||||
import { usePlatformsStore } from '../../stores/platforms.js'
|
||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import BrowserExtensionCard from '../settings/BrowserExtensionCard.vue'
|
||||
import ExtensionKeyBar from '../credentials/ExtensionKeyBar.vue'
|
||||
import CredentialUploadDialog from '../credentials/CredentialUploadDialog.vue'
|
||||
import CredentialCard from './CredentialCard.vue'
|
||||
@@ -240,6 +252,21 @@ async function saveDownloader() {
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* Same section-header language as Settings -> Maintenance/Cleanup (2026-07-02
|
||||
theme unification) so every admin surface reads identically. */
|
||||
.fc-section__title {
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.fc-section__hint {
|
||||
font-size: 0.8rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.fc-help {
|
||||
font-size: 12px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
<div>
|
||||
<SchedulerStatusBar :status="store.scheduleStatus" class="fc-subs__sched" />
|
||||
|
||||
<!-- Daily-use ordering (2026-07-02): what needs me → what came in →
|
||||
the full source list. Both cards render nothing when there's
|
||||
nothing to say. -->
|
||||
<NeedsAttentionCard />
|
||||
<RecentArrivalsCard />
|
||||
|
||||
<div class="fc-subs__bar">
|
||||
<v-btn color="accent" prepend-icon="mdi-plus" @click="openAddSource(null)">
|
||||
Add subscription
|
||||
@@ -319,6 +325,8 @@ import { useRoute, useRouter } from 'vue-router'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import { usePlatformsStore } from '../../stores/platforms.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import NeedsAttentionCard from './NeedsAttentionCard.vue'
|
||||
import RecentArrivalsCard from './RecentArrivalsCard.vue'
|
||||
import SourceRow from './SourceRow.vue'
|
||||
import SourceCard from './SourceCard.vue'
|
||||
import SourceHealthDot from './SourceHealthDot.vue'
|
||||
|
||||
@@ -101,8 +101,10 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
})
|
||||
}
|
||||
|
||||
// Destructive: deletes ALL general + character tags so the operator can
|
||||
// re-tag from scratch via auto-suggest. fandom + series preserved.
|
||||
// Destructive whole-instance reset: deletes ALL general + character tags AND
|
||||
// their applications (the heads' training data included) — fandom + series
|
||||
// preserved. dry-run returns a `confirm` token; the apply must pass it back
|
||||
// ({ dryRun: false, confirm }) or the server rejects it.
|
||||
function resetContentTagging(opts = {}) {
|
||||
return _dryRunPost('/api/admin/tags/reset-content', opts)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||
import { useSourcesStore } from './sources.js'
|
||||
|
||||
export const useDownloadsStore = defineStore('downloads', () => {
|
||||
const api = useApi()
|
||||
@@ -106,6 +107,44 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
return failing.value
|
||||
}
|
||||
|
||||
// --- Failing-source retries (shared by the Subscriptions needs-attention
|
||||
// card and the Downloads maintenance menu — one implementation for both
|
||||
// surfaces). Data-only: callers own the toasts, per this store's style.
|
||||
// A single deliberate retry forces past cooldown; BULK retries keep
|
||||
// cooldown enforcement ON so N failing sources on one platform don't all
|
||||
// retry into the very rate limit the cooldown is preventing.
|
||||
async function retrySource(source) {
|
||||
const sourcesStore = useSourcesStore()
|
||||
try {
|
||||
await sourcesStore.checkNow(source.id, { force: true })
|
||||
return 'queued'
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) return 'already_running'
|
||||
throw e
|
||||
} finally {
|
||||
await Promise.all([loadFailing(), loadStats(24)])
|
||||
}
|
||||
}
|
||||
|
||||
async function retryAllFailing(sources) {
|
||||
const sourcesStore = useSourcesStore()
|
||||
const tally = { ok: 0, deferred: 0, conflict: 0 }
|
||||
try {
|
||||
for (const s of sources) {
|
||||
try {
|
||||
const body = await sourcesStore.checkNow(s.id)
|
||||
if (body?.status === 'deferred') tally.deferred += 1
|
||||
else tally.ok += 1
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) tally.conflict += 1
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await Promise.all([loadFailing(), loadStats(24)])
|
||||
}
|
||||
return tally
|
||||
}
|
||||
|
||||
async function loadActive() {
|
||||
const [running, pending] = await Promise.all([
|
||||
api.get('/api/downloads', { params: { status: 'running', limit: 50 } }),
|
||||
@@ -128,5 +167,6 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
loadFirst, loadMore, loadOne, loadLastForSource, applyFilter,
|
||||
closeDetail, loadStats,
|
||||
loadActivity, loadFailing, loadActive, recoverStalled,
|
||||
retrySource, retryAllFailing,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { toast } from '../utils/toast.js'
|
||||
import { ref, computed } from 'vue'
|
||||
import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
// Import SETTINGS only. The manual-scan trigger + task-list surfaces retired
|
||||
// with the Import tab (2026-07-02): imports arrive via downloads/extension and
|
||||
// heal themselves (Layer-2 auto-refetch sweep); their runs show in Activity.
|
||||
// The /api/import/trigger + /tasks endpoints remain for direct/API use.
|
||||
// Consumers: ImportFiltersForm (Maintenance → Ingestion & filters) and the
|
||||
// Subscriptions Settings tab (downloader/extdl/schedule fields on the same
|
||||
// ImportSettings row).
|
||||
export const useImportStore = defineStore('import', () => {
|
||||
const api = useApi()
|
||||
|
||||
@@ -10,16 +17,6 @@ export const useImportStore = defineStore('import', () => {
|
||||
const settingsLoading = ref(false)
|
||||
const settingsError = ref(null)
|
||||
|
||||
const activeBatch = ref(null)
|
||||
const statusLoading = ref(false)
|
||||
|
||||
const tasks = ref([])
|
||||
const tasksNextCursor = ref(null)
|
||||
const tasksLoading = ref(false)
|
||||
const tasksFilter = ref({ status: null, limit: 50 })
|
||||
|
||||
const triggerError = ref(null)
|
||||
|
||||
async function loadSettings() {
|
||||
settingsLoading.value = true
|
||||
settingsError.value = null
|
||||
@@ -43,129 +40,8 @@ export const useImportStore = defineStore('import', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshStatus() {
|
||||
statusLoading.value = true
|
||||
try {
|
||||
const body = await api.get('/api/import/status')
|
||||
activeBatch.value = body.active_batch
|
||||
} finally {
|
||||
statusLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerScan(mode = 'quick') {
|
||||
if (!['quick', 'deep', 'verify'].includes(mode)) {
|
||||
throw new Error(`unsupported scan mode: ${mode}`)
|
||||
}
|
||||
triggerError.value = null
|
||||
try {
|
||||
await api.post('/api/import/trigger', { body: { mode } })
|
||||
// Acknowledge immediately so the click isn't invisible. scan_directory
|
||||
// can finalize the batch synchronously when every file in /import is
|
||||
// already on a non-failed ImportTask (operator-flagged 2026-05-25:
|
||||
// 233k existing tasks → all paths in skip-set → files_seen=0 →
|
||||
// batch flashes 'running' for <100ms then 'complete' before the
|
||||
// first refreshStatus() lands; UI never sees the active state).
|
||||
const label = mode === 'deep'
|
||||
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
|
||||
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
|
||||
toast({ text: label, type: 'success' })
|
||||
await refreshStatus()
|
||||
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
|
||||
// Operator-flagged 2026-05-25: the prior "no new files" message was
|
||||
// misleading because deep scan IS doing work (refresh) even when
|
||||
// there are no new files to import. Surface the real workload count
|
||||
// (imported + refreshed + queued) instead. For quick scan + zero
|
||||
// queued work, fall back to "up to date" instead of the old
|
||||
// implementation-detail-leaking message.
|
||||
setTimeout(async () => {
|
||||
await refreshStatus()
|
||||
await loadTasks(true)
|
||||
if (activeBatch.value || mode === 'verify') return
|
||||
// Batch finalized quickly; figure out what actually happened.
|
||||
// The task list was just refreshed; the freshest row(s) carry
|
||||
// the batch outcome.
|
||||
const batchId = tasks.value[0]?.batch_id
|
||||
const sameBatch = batchId
|
||||
? tasks.value.filter(t => t.batch_id === batchId)
|
||||
: []
|
||||
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
|
||||
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
|
||||
if (mode === 'deep' && sameBatch.length > 0) {
|
||||
toast({
|
||||
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
|
||||
type: 'info',
|
||||
})
|
||||
} else if (mode === 'quick' && sameBatch.length > 0) {
|
||||
toast({
|
||||
text: `Quick scan finished — ${newImported} new file(s) queued`,
|
||||
type: 'info',
|
||||
})
|
||||
} else {
|
||||
toast({ text: 'Library is up to date', type: 'info' })
|
||||
}
|
||||
}, 2000)
|
||||
} catch (e) {
|
||||
triggerError.value = e.message
|
||||
toast({ text: `Scan failed: ${e.message}`, type: 'error' })
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function loadTasks(reset = true) {
|
||||
tasksLoading.value = true
|
||||
try {
|
||||
const params = { limit: tasksFilter.value.limit }
|
||||
if (tasksFilter.value.status) params.status = tasksFilter.value.status
|
||||
if (!reset && tasksNextCursor.value) params.cursor = tasksNextCursor.value
|
||||
const body = await api.get('/api/import/tasks', { params })
|
||||
tasks.value = reset ? body.tasks : [...tasks.value, ...body.tasks]
|
||||
tasksNextCursor.value = body.next_cursor
|
||||
} finally {
|
||||
tasksLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function setStatusFilter(status) {
|
||||
tasksFilter.value.status = status
|
||||
}
|
||||
|
||||
async function retryFailed() {
|
||||
await api.post('/api/import/retry-failed')
|
||||
await loadTasks(true)
|
||||
}
|
||||
|
||||
async function clearCompleted(ageDays = 0) {
|
||||
await api.post('/api/import/clear-completed', { body: { age_days: ageDays } })
|
||||
await loadTasks(true)
|
||||
}
|
||||
|
||||
async function clearStuck() {
|
||||
const body = await api.post('/api/import/clear-stuck')
|
||||
await loadTasks(true)
|
||||
await refreshStatus()
|
||||
return body
|
||||
}
|
||||
|
||||
// Layer-2 one-shot re-download for a failed task's (corrupt) file.
|
||||
// Returns the endpoint's status dict (refetch_queued / no_source /
|
||||
// already_refetched). Caller surfaces it as a toast.
|
||||
async function refetchTask(taskId) {
|
||||
const body = await api.post(`/api/import/tasks/${taskId}/refetch`)
|
||||
await loadTasks(true)
|
||||
return body
|
||||
}
|
||||
|
||||
const hasMore = computed(() => tasksNextCursor.value !== null)
|
||||
|
||||
return {
|
||||
settings, settingsLoading, settingsError,
|
||||
activeBatch, statusLoading,
|
||||
tasks, tasksLoading, tasksFilter, hasMore,
|
||||
triggerError,
|
||||
loadSettings, patchSettings,
|
||||
refreshStatus, triggerScan,
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck,
|
||||
refetchTask,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -39,6 +39,17 @@
|
||||
<TagMaintenanceCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section fc-danger">
|
||||
<h3 class="fc-section__title fc-danger__title">Danger zone</h3>
|
||||
<p class="fc-section__hint">
|
||||
Whole-instance resets. Each needs a fresh preview and a typed
|
||||
confirmation code.
|
||||
</p>
|
||||
<div class="fc-tile-grid">
|
||||
<DangerZoneCard />
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -50,6 +61,7 @@ import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
|
||||
import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
|
||||
import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
|
||||
import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
|
||||
import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@@ -75,4 +87,12 @@ import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
|
||||
gap: 14px;
|
||||
align-items: start;
|
||||
}
|
||||
/* Danger zone: visually fenced off from routine cleanup so the reset can't
|
||||
be mistaken for maintenance. */
|
||||
.fc-danger {
|
||||
border-top: 1px solid rgba(var(--v-theme-error), 0.45);
|
||||
padding-top: 18px;
|
||||
margin-top: 36px;
|
||||
}
|
||||
.fc-danger__title { color: rgb(var(--v-theme-error)); }
|
||||
</style>
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
<template>
|
||||
<v-container class="pt-3 pb-8">
|
||||
<h1 class="fc-h1 mb-4">{{ title }}</h1>
|
||||
<v-alert type="info" variant="tonal" icon="mdi-toolbox">
|
||||
This surface is a placeholder. It will be implemented in
|
||||
<strong>{{ surfaceOwner }}</strong>.
|
||||
</v-alert>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
|
||||
const route = useRoute()
|
||||
const title = computed(() => route.meta.title ?? 'FabledCurator')
|
||||
|
||||
const surfaceOwner = computed(() => {
|
||||
const fc2 = new Set(['gallery', 'showcase', 'tags', 'settings'])
|
||||
const fc3 = new Set(['subscriptions', 'credentials', 'downloads'])
|
||||
if (fc2.has(route.name)) return 'FC-2 (image backbone)'
|
||||
if (fc3.has(route.name)) return 'FC-3 (subscription backbone)'
|
||||
return 'a future sub-project'
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-h1 {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 32px;
|
||||
font-weight: 500;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
</style>
|
||||
@@ -17,7 +17,6 @@
|
||||
>
|
||||
<v-tab value="overview">Overview</v-tab>
|
||||
<v-tab value="activity">Activity</v-tab>
|
||||
<v-tab value="import">Import</v-tab>
|
||||
<v-tab value="cleanup">Cleanup</v-tab>
|
||||
<v-tab value="maintenance">Maintenance</v-tab>
|
||||
</v-tabs>
|
||||
@@ -29,34 +28,24 @@
|
||||
class="mt-4"
|
||||
@open-activity="tab = 'activity'"
|
||||
/>
|
||||
<!-- Health strip (2026-07-02): the same GPU + downloads panels the
|
||||
Activity tab uses — Overview answers "is everything healthy?"
|
||||
across ALL systems, not just the Celery queues above. -->
|
||||
<div class="fc-health-row mt-4">
|
||||
<GpuActivityPanel @open-maintenance="tab = 'maintenance'" />
|
||||
<DownloadsActivityPanel />
|
||||
</div>
|
||||
<v-alert
|
||||
v-if="system.stats && (system.stats.tasks.pending + system.stats.tasks.queued) > 0"
|
||||
type="info" variant="tonal" class="mt-4" closable
|
||||
>
|
||||
{{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
|
||||
<v-btn variant="text" size="small" @click="tab = 'import'">Go to Import tab</v-btn>
|
||||
<v-btn variant="text" size="small" @click="tab = 'activity'">View in Activity</v-btn>
|
||||
</v-alert>
|
||||
<!-- Browser-extension install/download lives on Overview (moved
|
||||
from Maintenance 2026-05-25). Overview is the discovery
|
||||
surface for "things to set up"; Maintenance is for
|
||||
housekeeping of already-set-up systems. -->
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="activity">
|
||||
<SystemActivityTab />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="import">
|
||||
<!-- Order: filters → trigger → recent tasks. Filters hoisted above the
|
||||
trigger (operator-flagged 2026-06-04); the task list stays
|
||||
directly below the trigger so hit/miss feedback is adjacent to the
|
||||
button that produced it (operator-flagged 2026-05-25). -->
|
||||
<ImportFiltersForm />
|
||||
<v-divider class="my-6" />
|
||||
<ImportTriggerPanel />
|
||||
<v-divider class="my-6" />
|
||||
<ImportTaskList />
|
||||
<SystemActivityTab @open-maintenance="tab = 'maintenance'" />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="cleanup">
|
||||
@@ -73,21 +62,17 @@
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { useSystemStore } from '../stores/system.js'
|
||||
import { useImportStore } from '../stores/import.js'
|
||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||
import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
|
||||
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||
import GpuActivityPanel from '../components/settings/GpuActivityPanel.vue'
|
||||
import DownloadsActivityPanel from '../components/settings/DownloadsActivityPanel.vue'
|
||||
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
||||
import CleanupView from './CleanupView.vue'
|
||||
import { useMLStore } from '../stores/ml.js'
|
||||
|
||||
const tab = ref('overview')
|
||||
const system = useSystemStore()
|
||||
const importStore = useImportStore()
|
||||
const mlStore = useMLStore()
|
||||
|
||||
let pollId = null
|
||||
@@ -96,15 +81,7 @@ function startPolling() {
|
||||
if (pollId) return
|
||||
system.refreshStats()
|
||||
pollId = setInterval(() => {
|
||||
if (!document.hidden) {
|
||||
system.refreshStats()
|
||||
if (tab.value === 'import') {
|
||||
importStore.refreshStatus()
|
||||
// Refresh the task list while a batch is in flight so the UI
|
||||
// doesn't sit stale next to a ticking imported/skipped counter.
|
||||
if (importStore.activeBatch) importStore.loadTasks(true)
|
||||
}
|
||||
}
|
||||
if (!document.hidden) system.refreshStats()
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
@@ -112,21 +89,11 @@ function stopPolling() {
|
||||
if (pollId) { clearInterval(pollId); pollId = null }
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
startPolling()
|
||||
importStore.loadSettings()
|
||||
importStore.refreshStatus()
|
||||
importStore.loadTasks(true)
|
||||
})
|
||||
onMounted(startPolling)
|
||||
onUnmounted(stopPolling)
|
||||
|
||||
watch(tab, (t) => {
|
||||
if (t === 'import') {
|
||||
importStore.refreshStatus()
|
||||
importStore.loadTasks(true)
|
||||
} else if (t === 'maintenance') {
|
||||
mlStore.loadSettings()
|
||||
}
|
||||
if (t === 'maintenance') mlStore.loadSettings()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -135,4 +102,13 @@ watch(tab, (t) => {
|
||||
.fc-settings {
|
||||
max-width: 1140px;
|
||||
}
|
||||
/* Overview health strip: the two pulse panels side by side, stacking on
|
||||
narrow screens. Cancel the panels' own bottom margin — the row owns gaps. */
|
||||
.fc-health-row {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(340px, 1fr));
|
||||
gap: 16px;
|
||||
align-items: start;
|
||||
}
|
||||
.fc-health-row :deep(.v-card) { margin-bottom: 0 !important; }
|
||||
</style>
|
||||
|
||||
+31
-1
@@ -562,7 +562,7 @@ async def test_trigger_normalize_live_queues(client, monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_content_tagging_dry_run_returns_counts(client, db):
|
||||
async def test_reset_content_tagging_dry_run_returns_counts_and_token(client, db):
|
||||
db.add_all([
|
||||
Tag(name="solo", kind=TagKind.general),
|
||||
Tag(name="naruto", kind=TagKind.character),
|
||||
@@ -579,3 +579,33 @@ async def test_reset_content_tagging_dry_run_returns_counts(client, db):
|
||||
assert body["by_kind"] == {"general": 1, "character": 1}
|
||||
# dry-run leaves the rows in place — fandom + series untouched too.
|
||||
assert "deleted" not in body
|
||||
# The preview arms the apply: an 8-hex confirm token over the live counts.
|
||||
assert len(body["confirm"]) == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_content_tagging_apply_requires_confirm_token(client, db):
|
||||
from sqlalchemy import func, select
|
||||
|
||||
db.add(Tag(name="solo", kind=TagKind.general))
|
||||
await db.commit()
|
||||
|
||||
# No token / wrong token → rejected, nothing deleted.
|
||||
for bad in ({"dry_run": False}, {"dry_run": False, "confirm": "deadbeef"}):
|
||||
resp = await client.post("/api/admin/tags/reset-content", json=bad)
|
||||
assert resp.status_code == 400
|
||||
remaining = (await db.execute(
|
||||
select(func.count()).select_from(Tag)
|
||||
)).scalar_one()
|
||||
assert remaining == 1
|
||||
|
||||
# The token from a fresh preview unlocks the apply.
|
||||
token = (await (await client.post(
|
||||
"/api/admin/tags/reset-content", json={"dry_run": True}
|
||||
)).get_json())["confirm"]
|
||||
resp = await client.post(
|
||||
"/api/admin/tags/reset-content",
|
||||
json={"dry_run": False, "confirm": token},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert (await resp.get_json())["deleted"] == 1
|
||||
|
||||
+31
-1
@@ -127,7 +127,7 @@ async def test_backfill_enqueues_then_is_idempotent(db):
|
||||
await _img(db, "c" * 64)
|
||||
await _img(db, "d" * 64)
|
||||
await db.commit()
|
||||
from backend.app.tasks.ml import enqueue_gpu_backfill
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
n = enqueue_gpu_backfill("ccip") # sync task, own session
|
||||
assert n >= 2
|
||||
@@ -260,3 +260,33 @@ async def test_errors_endpoint_reports_triage_view(client, db):
|
||||
assert item["reason_class"] == "truncated_or_corrupt"
|
||||
assert item["triage_status"] is None
|
||||
assert item["image_url"].startswith("/images/")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cpu_embed_never_blocks_gpu_crop_backfills(db):
|
||||
"""B3 invariant (operator 2026-07-02): ccip (detect + character) and
|
||||
siglip (concept crops) completion is judged per-pipeline — gpu_job rows and
|
||||
image_region state — never inferred from image_record.siglip_embedding. So
|
||||
an image the CPU fallback already embedded still gets both crop jobs; only
|
||||
the whole-image 'embed' job (the SAME artifact the CPU path produces) is
|
||||
satisfied by it."""
|
||||
from backend.app.models import MLSettings
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
img = await _img(db, "7" * 64)
|
||||
cur = (await db.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
)).scalar_one()
|
||||
# As if the CPU fallback already embedded it under the current model.
|
||||
img.siglip_embedding = [0.1] * 1152
|
||||
img.siglip_model_version = cur
|
||||
await db.commit()
|
||||
|
||||
assert enqueue_gpu_backfill("ccip") == 1 # crops still open
|
||||
assert enqueue_gpu_backfill("siglip") == 1 # concept crops still open
|
||||
assert enqueue_gpu_backfill("embed") == 0 # same artifact — already done
|
||||
|
||||
tasks = set((await db.execute(
|
||||
select(GpuJob.task).where(GpuJob.image_record_id == img.id)
|
||||
)).scalars().all())
|
||||
assert tasks == {"ccip", "siglip"}
|
||||
|
||||
@@ -922,7 +922,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||
lambda image_id: thumb_calls.append(image_id),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ml_mod.tag_and_embed, "delay",
|
||||
ml_mod.embed_image, "delay",
|
||||
lambda image_id: ml_calls.append(image_id),
|
||||
)
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ def test_refetch_same_link_keeps_canonical_file(db_sync, tmp_path, monkeypatch):
|
||||
|
||||
import backend.app.tasks.ml as ml_mod
|
||||
import backend.app.tasks.thumbnail as thumb_mod
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda i: None)
|
||||
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
|
||||
|
||||
out = ext.fetch_external_link(link_id)
|
||||
@@ -234,7 +234,7 @@ def test_downloaded_archive_gets_provenance_and_tagging(db_sync, tmp_path, monke
|
||||
tagged, thumbed = [], []
|
||||
import backend.app.tasks.ml as ml_mod
|
||||
import backend.app.tasks.thumbnail as thumb_mod
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda i: tagged.append(i))
|
||||
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: tagged.append(i))
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: thumbed.append(i))
|
||||
|
||||
out = ext.fetch_external_link(link_id)
|
||||
|
||||
@@ -30,7 +30,7 @@ async def test_enqueue_siglip_backfill_gates_on_concept_region(db):
|
||||
# back-catalogue) and skips ones that already have one — and never double-
|
||||
# enqueues an image that already has a pending siglip job.
|
||||
from backend.app.models import MLSettings
|
||||
from backend.app.tasks.ml import enqueue_gpu_backfill
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
cur = (await db.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
@@ -71,7 +71,7 @@ async def test_enqueue_embed_backfill_selects_stale_and_unembedded(db):
|
||||
# stamped under a DIFFERENT model version (an operator swap); skip ones
|
||||
# already at the current version.
|
||||
from backend.app.models import MLSettings
|
||||
from backend.app.tasks.ml import enqueue_gpu_backfill
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
cur = (await db.execute(
|
||||
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||
@@ -99,7 +99,7 @@ async def test_enqueue_embed_backfill_selects_stale_and_unembedded(db):
|
||||
async def test_reprocess_resets_done_jobs_to_pending(db):
|
||||
# Re-process (#1202): done/error jobs of a task go back to pending so the
|
||||
# agent re-runs the whole library under the current pipeline.
|
||||
from backend.app.tasks.ml import reprocess_gpu_jobs
|
||||
from backend.app.tasks.gpu_queue import reprocess_gpu_jobs
|
||||
|
||||
img = await _img(db, "r1" * 32)
|
||||
job = await GpuJobService(db).enqueue(img.id, "ccip")
|
||||
@@ -274,7 +274,7 @@ async def test_backfill_skips_errored_images(db):
|
||||
# An errored job is a TOMBSTONE for its (image, task): no backfill variant
|
||||
# re-enqueues it — retry is deliberate-only (/retry_errors). Pre-fix, the
|
||||
# hourly ccip run minted a fresh doomed job per bad file forever.
|
||||
from backend.app.tasks.ml import enqueue_gpu_backfill
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
img = await _img(db, "f1" * 32)
|
||||
svc = GpuJobService(db)
|
||||
@@ -294,7 +294,7 @@ async def test_backfill_prunes_moot_error_tombstones(db):
|
||||
# Loop-era duplicates: several error rows for one (image, task), all made
|
||||
# moot by a later done row. The backfill's dedupe pass removes them, and
|
||||
# the done row still blocks re-enqueue.
|
||||
from backend.app.tasks.ml import enqueue_gpu_backfill
|
||||
from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
img = await _img(db, "f2" * 32)
|
||||
for i in range(3):
|
||||
|
||||
@@ -116,6 +116,30 @@ def test_transparent_filter(importer, import_layout):
|
||||
assert result.skip_reason == SkipReason.too_transparent
|
||||
|
||||
|
||||
def test_single_color_filter(importer, import_layout):
|
||||
"""The skip_single_color setting existed since FC-2 but was never wired
|
||||
(the audit module's docstring said so); wired 2026-07-02 using the same
|
||||
canonical predicate as the Cleanup audit. Solid fill skips when enabled,
|
||||
imports when disabled (the default)."""
|
||||
from PIL import Image as PILImage
|
||||
|
||||
import_root, _ = import_layout
|
||||
solid = import_root / "solid.png"
|
||||
solid.parent.mkdir(parents=True, exist_ok=True)
|
||||
PILImage.new("RGB", (100, 100), (12, 34, 56)).save(solid)
|
||||
|
||||
importer.settings.skip_single_color = True
|
||||
importer.settings.single_color_threshold = 0.95
|
||||
result = importer.import_one(solid)
|
||||
assert result.status == "skipped"
|
||||
assert result.skip_reason == SkipReason.single_color
|
||||
|
||||
importer.settings.skip_single_color = False
|
||||
solid2 = import_root / "solid2.png"
|
||||
PILImage.new("RGB", (100, 100), (200, 10, 10)).save(solid2)
|
||||
assert importer.import_one(solid2).status == "imported"
|
||||
|
||||
|
||||
def test_unsupported_extension(importer, import_layout):
|
||||
# FC-2d-iii: non-media is no longer skipped — it's captured as a
|
||||
# PostAttachment so nothing a post contained is lost.
|
||||
|
||||
@@ -310,7 +310,7 @@ def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
|
||||
|
||||
|
||||
def test_recover_stalled_task_runs_ml_queue_uses_longer_threshold(db_sync):
|
||||
"""ml-queue tasks (tag_and_embed video branch) legitimately run
|
||||
"""ml-queue tasks (embed_image video branch) legitimately run
|
||||
past the default 5-min threshold. The sweep must NOT flag an
|
||||
ml-queue task that's only been running 10 min — the override
|
||||
threshold (25 min via QUEUE_STUCK_THRESHOLD_MINUTES) protects
|
||||
|
||||
@@ -46,7 +46,7 @@ def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch)
|
||||
|
||||
# No broker in this path — the post-import enqueue is best-effort anyway.
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
images_root.mkdir()
|
||||
@@ -116,7 +116,7 @@ def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
|
||||
from backend.app.tasks import thumbnail as thumb_mod
|
||||
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
images_root.mkdir()
|
||||
|
||||
+34
-2
@@ -1,4 +1,4 @@
|
||||
"""tag_and_embed (embed-only) / backfill task tests. The pure _is_video helper
|
||||
"""embed_image (embed-only) / backfill task tests. The pure _is_video helper
|
||||
is a unit test; the DB-touching backfill query is an integration test with
|
||||
monkeypatched dispatch."""
|
||||
|
||||
@@ -23,7 +23,7 @@ async def test_backfill_enqueues_missing(db, monkeypatch):
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
ml_tasks.tag_and_embed, "delay", lambda image_id: calls.append(image_id)
|
||||
ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
|
||||
)
|
||||
|
||||
img = ImageRecord(
|
||||
@@ -38,3 +38,35 @@ async def test_backfill_enqueues_missing(db, monkeypatch):
|
||||
count = ml_tasks.backfill()
|
||||
assert count >= 1
|
||||
assert img.id in calls
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_respects_cpu_embed_toggle(db, monkeypatch):
|
||||
"""B3: with cpu_embed_enabled off (agent-equipped stack, no ml-worker),
|
||||
the CPU backfill is a no-op — the GPU 'embed' backfill owns whole-image
|
||||
embeds there. Same gate the import hooks consult before dispatching."""
|
||||
from sqlalchemy import update
|
||||
|
||||
from backend.app.models import ImageRecord, MLSettings
|
||||
from backend.app.tasks import ml as ml_tasks
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
|
||||
)
|
||||
db.add(ImageRecord(
|
||||
path="/images/o.jpg", sha256="o" * 64, size_bytes=1,
|
||||
mime="image/jpeg", width=1, height=1,
|
||||
origin="imported_filesystem", integrity_status="unknown",
|
||||
siglip_embedding=None,
|
||||
))
|
||||
await db.execute(
|
||||
update(MLSettings).where(MLSettings.id == 1)
|
||||
.values(cpu_embed_enabled=False)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
assert ml_tasks.cpu_embed_enabled() is False
|
||||
assert ml_tasks.backfill() == 0
|
||||
assert calls == []
|
||||
|
||||
Reference in New Issue
Block a user