diff --git a/alembic/versions/0053_ml_settings_video_tagging.py b/alembic/versions/0053_ml_settings_video_tagging.py
new file mode 100644
index 0000000..1f192a4
--- /dev/null
+++ b/alembic/versions/0053_ml_settings_video_tagging.py
@@ -0,0 +1,49 @@
+"""ml_settings: video tagging knobs (cadence sampling + noise floor)
+
+#747. Video tag quality/perf: sample frames at a fixed cadence (interval) so a
+tag's frame-presence reflects real screen time, cap total frames so long videos
+stay bounded, and keep a tag only if it appears in >= min_tag_frames sampled
+frames. Operator-tunable via Settings → ML (replaces the VIDEO_ML_FRAMES env var).
+
+Revision ID: 0053
+Revises: 0052
+Create Date: 2026-06-16
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0053"
+down_revision: Union[str, None] = "0052"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "ml_settings",
+ sa.Column(
+ "video_frame_interval_seconds", sa.Float(), nullable=False,
+ server_default="4.0",
+ ),
+ )
+ op.add_column(
+ "ml_settings",
+ sa.Column(
+ "video_max_frames", sa.Integer(), nullable=False, server_default="64",
+ ),
+ )
+ op.add_column(
+ "ml_settings",
+ sa.Column(
+ "video_min_tag_frames", sa.Integer(), nullable=False,
+ server_default="3",
+ ),
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("ml_settings", "video_min_tag_frames")
+ op.drop_column("ml_settings", "video_max_frames")
+ op.drop_column("ml_settings", "video_frame_interval_seconds")
diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py
index ef3d2fa..b3e8d94 100644
--- a/backend/app/api/ml_admin.py
+++ b/backend/app/api/ml_admin.py
@@ -14,6 +14,9 @@ _EDITABLE = (
"centroid_similarity_threshold",
"min_reference_images",
"tagger_store_floor",
+ "video_frame_interval_seconds",
+ "video_max_frames",
+ "video_min_tag_frames",
)
@@ -32,6 +35,9 @@ async def get_settings():
"centroid_similarity_threshold": s.centroid_similarity_threshold,
"min_reference_images": s.min_reference_images,
"tagger_store_floor": s.tagger_store_floor,
+ "video_frame_interval_seconds": s.video_frame_interval_seconds,
+ "video_max_frames": s.video_max_frames,
+ "video_min_tag_frames": s.video_min_tag_frames,
"tagger_model_version": s.tagger_model_version,
"embedder_model_version": s.embedder_model_version,
}
@@ -85,6 +91,15 @@ def _validate(p: dict) -> str | None:
f"suggestion_threshold_{cat} cannot be below tagger_store_floor "
f"({floor}) — predictions below the floor are not stored"
)
+ # Video tagging (#747).
+ if p["video_frame_interval_seconds"] <= 0:
+ return "video_frame_interval_seconds must be > 0"
+ if p["video_max_frames"] < 1:
+ return "video_max_frames must be >= 1"
+ if p["video_min_tag_frames"] < 1:
+ return "video_min_tag_frames must be >= 1"
+ if p["video_min_tag_frames"] > p["video_max_frames"]:
+ return "video_min_tag_frames cannot exceed video_max_frames"
return None
diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py
index e28686d..3ae5a1b 100644
--- a/backend/app/models/ml_settings.py
+++ b/backend/app/models/ml_settings.py
@@ -40,6 +40,21 @@ class MLSettings(Base):
min_reference_images: Mapped[int] = mapped_column(
Integer, nullable=False, default=5
)
+ # Video tagging (#747). Sample one frame every N seconds (fixed CADENCE, not a
+ # fixed count) so a tag's frame-presence reflects real screen time regardless
+ # of video length; cap the total so a long video can't explode into hundreds
+ # of inferences (the cadence stretches past the cap). A tag is kept only if it
+ # appears in >= video_min_tag_frames sampled frames (≈ that many × interval
+ # seconds on screen) — duration-independent noise rejection. Operator-tunable.
+ video_frame_interval_seconds: Mapped[float] = mapped_column(
+ Float, nullable=False, default=4.0
+ )
+ video_max_frames: Mapped[int] = mapped_column(
+ Integer, nullable=False, default=64
+ )
+ video_min_tag_frames: Mapped[int] = mapped_column(
+ Integer, nullable=False, default=3
+ )
tagger_model_version: Mapped[str] = mapped_column(
String(128), nullable=False, default="camie-tagger-v2"
)
diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py
index 05d1581..7017e9c 100644
--- a/backend/app/tasks/ml.py
+++ b/backend/app/tasks/ml.py
@@ -49,11 +49,12 @@ def tag_and_embed(self, image_id: int) -> dict:
"""Run Camie + SigLIP on one image; store predictions + embedding;
then enqueue per-image allowlist application.
- Video: sample frames between 10% and 90% of duration (VIDEO_ML_FRAMES,
- default 6). Max-pool tagger confidences across frames, mean-pool the
+ Video (#747): sample frames at a fixed cadence (ml_settings
+ video_frame_interval_seconds, capped at video_max_frames), keep a tag only if
+ it appears in >= video_min_tag_frames frames and average its confidence over
+ those frames (mean-pool, not max — kills one-frame noise); mean-pool the
SigLIP embeddings. On no-frames returns status='no_frames' (not an error).
"""
- import os
import time
from ..services.ml.embedder import get_embedder
@@ -116,7 +117,9 @@ def tag_and_embed(self, image_id: int) -> dict:
phase = "video_sample_frames"
t0 = time.monotonic()
frames = _sample_video_frames(
- src, int(os.environ.get("VIDEO_ML_FRAMES", "6"))
+ src,
+ interval=settings.video_frame_interval_seconds,
+ max_frames=settings.video_max_frames,
)
log.info(
"tag_and_embed sampled %d frame(s) in %.1fs: %s",
@@ -127,13 +130,19 @@ def tag_and_embed(self, image_id: int) -> dict:
phase = "video_infer"
import numpy as np
- preds = _maxpool_predictions(
+ preds = _aggregate_video_predictions(
[tagger.infer(f, store_floor=settings.tagger_store_floor)
- for f in frames]
+ for f in frames],
+ min_frames=settings.video_min_tag_frames,
)
embedding = np.mean(
[embedder.infer(f) for f in frames], axis=0
).astype("float32")
+ log.info(
+ "tag_and_embed video aggregated %d tag(s) from %d frame(s) "
+ "(min_frames=%d): %s",
+ len(preds), len(frames), settings.video_min_tag_frames, ctx,
+ )
for f in frames:
f.unlink(missing_ok=True)
else:
@@ -208,9 +217,17 @@ def tag_and_embed(self, image_id: int) -> dict:
return {"status": "ok", "image_id": image_id, "tags": len(preds)}
-def _sample_video_frames(src: Path, n: int) -> list[Path]:
- """Extract n frames evenly between 10% and 90% of duration via ffmpeg.
- Returns temp file paths (caller deletes). Empty list on failure."""
+def _sample_video_frames(
+ src: Path, *, interval: float, max_frames: int,
+) -> list[Path]:
+ """Sample frames at a fixed CADENCE — one every `interval` seconds — so a
+ tag's frame-presence reflects real screen time regardless of video length
+ (#747). The count is capped at `max_frames`: a video longer than
+ interval×max_frames stretches the spacing instead of exploding the frame
+ count (keeps cost bounded so a long video can't hog the single ml-worker).
+ Frames are taken across the 5%–95% window (skip intro/outro black/cards) via
+ per-frame fast-seek. Returns temp file paths (caller deletes); [] on failure.
+ """
import json
import subprocess
import tempfile
@@ -229,20 +246,25 @@ def _sample_video_frames(src: Path, n: int) -> list[Path]:
if duration <= 0:
return []
- start, end = duration * 0.10, duration * 0.90
- step = (end - start) / max(n - 1, 1)
+ start, end = duration * 0.05, duration * 0.95
+ span = max(end - start, 0.0)
+ # Cadence count, clamped to [1, max_frames]. int(span/interval)+1 ≈ one frame
+ # per `interval` seconds across the window; the cap stretches spacing on very
+ # long videos.
+ n = max(1, min(int(span / interval) + 1, max(1, max_frames)))
+ step = span / max(n - 1, 1)
out: list[Path] = []
tmpdir = Path(tempfile.mkdtemp(prefix="fc_vid_"))
for i in range(n):
ts = start + i * step
- dest = tmpdir / f"frame_{i:02d}.jpg"
+ dest = tmpdir / f"frame_{i:04d}.jpg"
try:
subprocess.run(
[
"ffmpeg", "-ss", f"{ts:.2f}", "-i", str(src),
"-frames:v", "1", "-q:v", "3", "-y", str(dest),
],
- check=True, capture_output=True, timeout=60,
+ check=True, capture_output=True, timeout=30,
)
if dest.is_file():
out.append(dest)
@@ -251,18 +273,38 @@ def _sample_video_frames(src: Path, n: int) -> list[Path]:
return out
-def _maxpool_predictions(per_frame: list[dict]) -> dict:
- """Aggregate per-frame {name: TagPrediction} dicts by max confidence."""
- merged: dict[str, dict] = {}
+def _aggregate_video_predictions(per_frame: list[dict], *, min_frames: int) -> dict:
+ """Aggregate per-frame {name: TagPrediction} into one prediction set (#747).
+
+ A tag is kept only if it appears (≥ the tagger store floor, already applied)
+ in at least `min_frames` of the sampled frames — because sampling is at a
+ fixed cadence, that means it was on screen for roughly min_frames×interval
+ seconds, so a single-frame flicker / scene-transition artifact is dropped
+ while a genuine scene-local tag in a long video survives. Confidence is the
+ MEAN over the frames where the tag appears (not max — max re-inflated the
+ one-frame noise this whole change exists to remove).
+
+ `min_frames` is clamped to the number of frames actually sampled so a very
+ short video (1–2 frames) still tags instead of dropping everything.
+ """
+ n = len(per_frame)
+ if n == 0:
+ return {}
+ threshold = max(1, min(min_frames, n))
+ agg: dict[str, dict] = {}
for frame_preds in per_frame:
for name, p in frame_preds.items():
- cur = merged.get(name)
- if cur is None or p.confidence > cur["confidence"]:
- merged[name] = {
- "category": p.category,
- "confidence": p.confidence,
- }
- return merged
+ cur = agg.get(name)
+ if cur is None:
+ agg[name] = {"category": p.category, "sum": p.confidence, "count": 1}
+ else:
+ cur["sum"] += p.confidence
+ cur["count"] += 1
+ return {
+ name: {"category": v["category"], "confidence": v["sum"] / v["count"]}
+ for name, v in agg.items()
+ if v["count"] >= threshold
+ }
@celery.task(name="backend.app.tasks.ml.backfill", bind=True)
diff --git a/frontend/src/components/settings/MLThresholdSliders.vue b/frontend/src/components/settings/MLThresholdSliders.vue
index 11bc76b..f0028ca 100644
--- a/frontend/src/components/settings/MLThresholdSliders.vue
+++ b/frontend/src/components/settings/MLThresholdSliders.vue
@@ -30,6 +30,39 @@
+
+
+
+ Video tagging
+
+ Videos are tagged by sampling frames at a fixed cadence. A tag is kept
+ only if it shows up in enough frames (≈ that many × the interval in
+ seconds of screen time), which filters one-frame noise without losing
+ tags that only appear in part of a longer video.
+
+
+
+
+
+
+
+
+
+
+
+
@@ -59,9 +92,14 @@ async function save() {
const floor = local.tagger_store_floor
local.suggestion_threshold_character = Math.max(local.suggestion_threshold_character, floor)
local.suggestion_threshold_general = Math.max(local.suggestion_threshold_general, floor)
+ // Mirror the server invariant: a tag can't require more frames than are sampled.
+ local.video_min_tag_frames = Math.min(local.video_min_tag_frames, local.video_max_frames)
const patch = {}
for (const f of fields) patch[f.key] = local[f.key]
patch.tagger_store_floor = local.tagger_store_floor
+ patch.video_frame_interval_seconds = local.video_frame_interval_seconds
+ patch.video_max_frames = local.video_max_frames
+ patch.video_min_tag_frames = local.video_min_tag_frames
try { await store.patchSettings(patch) }
catch (e) { toast({ text: e.message, type: 'error' }) }
}
diff --git a/tests/test_api_ml_admin.py b/tests/test_api_ml_admin.py
index 76b1002..e46be48 100644
--- a/tests/test_api_ml_admin.py
+++ b/tests/test_api_ml_admin.py
@@ -56,6 +56,36 @@ async def test_suggestion_threshold_below_store_floor_rejected(client):
assert "tagger_store_floor" in (await resp.get_json())["error"]
+@pytest.mark.asyncio
+async def test_video_tagging_settings_default_and_patch(client):
+ """#747: video cadence/noise knobs are exposed + patchable."""
+ body = await (await client.get("/api/ml/settings")).get_json()
+ assert body["video_frame_interval_seconds"] == pytest.approx(4.0)
+ assert body["video_max_frames"] == 64
+ assert body["video_min_tag_frames"] == 3
+
+ resp = await client.patch(
+ "/api/ml/settings",
+ json={"video_frame_interval_seconds": 5, "video_max_frames": 40,
+ "video_min_tag_frames": 4},
+ )
+ assert resp.status_code == 200
+ out = await resp.get_json()
+ assert out["video_frame_interval_seconds"] == pytest.approx(5.0)
+ assert out["video_max_frames"] == 40
+ assert out["video_min_tag_frames"] == 4
+
+
+@pytest.mark.asyncio
+async def test_video_min_tag_frames_above_max_rejected(client):
+ resp = await client.patch(
+ "/api/ml/settings",
+ json={"video_max_frames": 10, "video_min_tag_frames": 20},
+ )
+ assert resp.status_code == 400
+ assert "video_min_tag_frames" in (await resp.get_json())["error"]
+
+
@pytest.mark.asyncio
async def test_backfill_and_recompute_trigger(client):
r1 = await client.post("/api/ml/backfill")
diff --git a/tests/test_tasks_ml.py b/tests/test_tasks_ml.py
index 3577360..62e5c5b 100644
--- a/tests/test_tasks_ml.py
+++ b/tests/test_tasks_ml.py
@@ -1,6 +1,6 @@
"""tag_and_embed / backfill task tests. Models aren't in CI, so we test
-the pure helpers (_maxpool_predictions, _is_video) as unit tests, and the
-DB-touching backfill query as an integration test with monkeypatched
+the pure helpers (_aggregate_video_predictions, _is_video) as unit tests, and
+the DB-touching backfill query as an integration test with monkeypatched
inference.
"""
@@ -9,7 +9,7 @@ from pathlib import Path
import pytest
from backend.app.services.ml.tagger import TagPrediction
-from backend.app.tasks.ml import _is_video, _maxpool_predictions
+from backend.app.tasks.ml import _aggregate_video_predictions, _is_video
def test_is_video():
@@ -18,15 +18,32 @@ def test_is_video():
assert _is_video(Path("a.jpg")) is False
-def test_maxpool_predictions():
- f1 = {"smile": TagPrediction("smile", "general", 0.6)}
- f2 = {
- "smile": TagPrediction("smile", "general", 0.9),
- "sword": TagPrediction("sword", "general", 0.7),
- }
- merged = _maxpool_predictions([f1, f2])
- assert merged["smile"]["confidence"] == 0.9
- assert merged["sword"]["confidence"] == 0.7
+def _pred(name, conf, cat="general"):
+ return {name: TagPrediction(name, cat, conf)}
+
+
+def test_aggregate_video_keeps_corroborated_and_means():
+ # #747: 4 frames; "smile" in 3, "sword" in 1 (noise). min_frames=2.
+ per_frame = [
+ {"smile": TagPrediction("smile", "general", 0.6),
+ "sword": TagPrediction("sword", "general", 0.9)},
+ _pred("smile", 0.8),
+ _pred("smile", 0.7),
+ {},
+ ]
+ out = _aggregate_video_predictions(per_frame, min_frames=2)
+ assert "sword" not in out # one-frame flicker dropped
+ assert abs(out["smile"]["confidence"] - (0.6 + 0.8 + 0.7) / 3) < 1e-9 # mean, not max
+
+
+def test_aggregate_video_clamps_min_frames_to_sample_count():
+ # Short video: 1 frame but min_frames=3 — clamp so it still tags.
+ out = _aggregate_video_predictions([_pred("solo", 0.8)], min_frames=3)
+ assert out["solo"]["confidence"] == 0.8
+
+
+def test_aggregate_video_empty():
+ assert _aggregate_video_predictions([], min_frames=3) == {}
@pytest.mark.integration