feat(ml): cadence-based video frame sampling + min-frame tag aggregation (#747)
Video tag noise root cause: frames were a FIXED count (6) max-pooled — a tag firing on one frame survived at peak confidence, and a fixed count under-samples long multi-scene videos so real scene-local tags looked like noise. Redesign (operator-steered): - Sample at a fixed CADENCE — one frame every `video_frame_interval_seconds` (default 4) across the 5–95% window — so a tag's frame-presence reflects real screen time independent of video length. Capped at `video_max_frames` (default 64): a long video stretches the spacing instead of exploding into hundreds of inferences, bounding per-video cost on the single ml-worker (per-frame ffmpeg timeout also cut 60s→30s). - Aggregate with `_aggregate_video_predictions`: keep a tag only if it appears in >= `video_min_tag_frames` sampled frames (≈ that many × interval seconds on screen — duration-independent noise rejection), with confidence = MEAN over the frames it appears in (not max). Clamps the threshold to the sample count so a 1–2-frame short video still tags. - All three knobs are DB-backed ml_settings (migration 0053), patchable via /api/ml/settings + sliders in the ML settings card — replaces the VIDEO_ML_FRAMES env var (product-not-project). Tests: aggregation drops one-frame noise + means corroborated tags + clamps on short videos; settings round-trip + min>max validation. Replaced the _maxpool_predictions unit test. NOTE: this is the QUALITY half of #747. The perf half — the ml-worker runs CPU-only — is GPU enablement, tracked separately in #872. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -56,6 +56,36 @@ async def test_suggestion_threshold_below_store_floor_rejected(client):
|
||||
assert "tagger_store_floor" in (await resp.get_json())["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_tagging_settings_default_and_patch(client):
|
||||
"""#747: video cadence/noise knobs are exposed + patchable."""
|
||||
body = await (await client.get("/api/ml/settings")).get_json()
|
||||
assert body["video_frame_interval_seconds"] == pytest.approx(4.0)
|
||||
assert body["video_max_frames"] == 64
|
||||
assert body["video_min_tag_frames"] == 3
|
||||
|
||||
resp = await client.patch(
|
||||
"/api/ml/settings",
|
||||
json={"video_frame_interval_seconds": 5, "video_max_frames": 40,
|
||||
"video_min_tag_frames": 4},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
out = await resp.get_json()
|
||||
assert out["video_frame_interval_seconds"] == pytest.approx(5.0)
|
||||
assert out["video_max_frames"] == 40
|
||||
assert out["video_min_tag_frames"] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_min_tag_frames_above_max_rejected(client):
|
||||
resp = await client.patch(
|
||||
"/api/ml/settings",
|
||||
json={"video_max_frames": 10, "video_min_tag_frames": 20},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "video_min_tag_frames" in (await resp.get_json())["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_and_recompute_trigger(client):
|
||||
r1 = await client.post("/api/ml/backfill")
|
||||
|
||||
+29
-12
@@ -1,6 +1,6 @@
|
||||
"""tag_and_embed / backfill task tests. Models aren't in CI, so we test
|
||||
the pure helpers (_maxpool_predictions, _is_video) as unit tests, and the
|
||||
DB-touching backfill query as an integration test with monkeypatched
|
||||
the pure helpers (_aggregate_video_predictions, _is_video) as unit tests, and
|
||||
the DB-touching backfill query as an integration test with monkeypatched
|
||||
inference.
|
||||
"""
|
||||
|
||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from backend.app.services.ml.tagger import TagPrediction
|
||||
from backend.app.tasks.ml import _is_video, _maxpool_predictions
|
||||
from backend.app.tasks.ml import _aggregate_video_predictions, _is_video
|
||||
|
||||
|
||||
def test_is_video():
|
||||
@@ -18,15 +18,32 @@ def test_is_video():
|
||||
assert _is_video(Path("a.jpg")) is False
|
||||
|
||||
|
||||
def test_maxpool_predictions():
|
||||
f1 = {"smile": TagPrediction("smile", "general", 0.6)}
|
||||
f2 = {
|
||||
"smile": TagPrediction("smile", "general", 0.9),
|
||||
"sword": TagPrediction("sword", "general", 0.7),
|
||||
}
|
||||
merged = _maxpool_predictions([f1, f2])
|
||||
assert merged["smile"]["confidence"] == 0.9
|
||||
assert merged["sword"]["confidence"] == 0.7
|
||||
def _pred(name, conf, cat="general"):
|
||||
return {name: TagPrediction(name, cat, conf)}
|
||||
|
||||
|
||||
def test_aggregate_video_keeps_corroborated_and_means():
|
||||
# #747: 4 frames; "smile" in 3, "sword" in 1 (noise). min_frames=2.
|
||||
per_frame = [
|
||||
{"smile": TagPrediction("smile", "general", 0.6),
|
||||
"sword": TagPrediction("sword", "general", 0.9)},
|
||||
_pred("smile", 0.8),
|
||||
_pred("smile", 0.7),
|
||||
{},
|
||||
]
|
||||
out = _aggregate_video_predictions(per_frame, min_frames=2)
|
||||
assert "sword" not in out # one-frame flicker dropped
|
||||
assert abs(out["smile"]["confidence"] - (0.6 + 0.8 + 0.7) / 3) < 1e-9 # mean, not max
|
||||
|
||||
|
||||
def test_aggregate_video_clamps_min_frames_to_sample_count():
|
||||
# Short video: 1 frame but min_frames=3 — clamp so it still tags.
|
||||
out = _aggregate_video_predictions([_pred("solo", 0.8)], min_frames=3)
|
||||
assert out["solo"]["confidence"] == 0.8
|
||||
|
||||
|
||||
def test_aggregate_video_empty():
|
||||
assert _aggregate_video_predictions([], min_frames=3) == {}
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
Reference in New Issue
Block a user