bvandeusen
19b962f1a7
The ml-worker's ONLY processing role is now the CPU whole-image embed fallback (tag_and_embed renamed embed_image — Camie tagging was retired #1189 and the name kept implying otherwise; videos were already handled agent-style: frame sampling + mean-pool). Detection/cropping/CCIP stay GPU-agent-only, and their completion is judged per-pipeline: ccip by gpu_job rows, siglip by concept regions at the current model version — never by image_record.siglip_embedding. A CPU embed therefore can NEVER close crop work for the agent (regression test pins this; only the whole-image 'embed' job, the same artifact, is satisfied). Making removal actually safe (operator will drop the container): - GPU-queue coordination (enqueue_gpu_backfill, recover_orphaned_gpu_jobs, reprocess_gpu_jobs) moved verbatim to tasks/gpu_queue.py on the maintenance quick lane — it lived on the 'ml' queue only by module colocation, which made the ml-worker a hard dependency of the whole agent pipeline. - New ml_settings.cpu_embed_enabled (migration 0074, default ON so agent-less installs keep working): OFF stops the four import hooks queueing embed work nothing will consume and no-ops the manual backfill; switch lives on the renamed 'CPU embedding backfill' card. - NB heads training / auto-apply still run on the ml image (sklearn) — a stack that removes the container gives those up too. Deploy note: in-flight messages under the old task names are dropped by the new workers; the 60s orphan sweep + hourly backfill re-fire under the new names immediately. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
73 lines
2.2 KiB
Python
73 lines
2.2 KiB
Python
"""embed_image (embed-only) / backfill task tests. The pure _is_video helper
|
|
is a unit test; the DB-touching backfill query is an integration test with
|
|
monkeypatched dispatch."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from backend.app.tasks.ml import _is_video
|
|
|
|
|
|
def test_is_video():
|
|
assert _is_video(Path("a.mp4")) is True
|
|
assert _is_video(Path("a.MKV")) is True
|
|
assert _is_video(Path("a.jpg")) is False
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_enqueues_missing(db, monkeypatch):
|
|
from backend.app.models import ImageRecord
|
|
from backend.app.tasks import ml as ml_tasks
|
|
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
|
|
)
|
|
|
|
img = ImageRecord(
|
|
path="/images/n.jpg", sha256="n" * 64, size_bytes=1,
|
|
mime="image/jpeg", width=1, height=1,
|
|
origin="imported_filesystem", integrity_status="unknown",
|
|
siglip_embedding=None,
|
|
)
|
|
db.add(img)
|
|
await db.commit()
|
|
|
|
count = ml_tasks.backfill()
|
|
assert count >= 1
|
|
assert img.id in calls
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_respects_cpu_embed_toggle(db, monkeypatch):
|
|
"""B3: with cpu_embed_enabled off (agent-equipped stack, no ml-worker),
|
|
the CPU backfill is a no-op — the GPU 'embed' backfill owns whole-image
|
|
embeds there. Same gate the import hooks consult before dispatching."""
|
|
from sqlalchemy import update
|
|
|
|
from backend.app.models import ImageRecord, MLSettings
|
|
from backend.app.tasks import ml as ml_tasks
|
|
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
|
|
)
|
|
db.add(ImageRecord(
|
|
path="/images/o.jpg", sha256="o" * 64, size_bytes=1,
|
|
mime="image/jpeg", width=1, height=1,
|
|
origin="imported_filesystem", integrity_status="unknown",
|
|
siglip_embedding=None,
|
|
))
|
|
await db.execute(
|
|
update(MLSettings).where(MLSettings.id == 1)
|
|
.values(cpu_embed_enabled=False)
|
|
)
|
|
await db.commit()
|
|
|
|
assert ml_tasks.cpu_embed_enabled() is False
|
|
assert ml_tasks.backfill() == 0
|
|
assert calls == []
|