From 7cdce0c47491e8441aecb05b61381012a952847b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 1 Jul 2026 00:35:03 -0400 Subject: [PATCH] =?UTF-8?q?feat(agent):=20temporal=20video=20dedup=20?= =?UTF-8?q?=E2=80=94=20drop=20near-duplicate=20frames=20before=20the=20GPU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Near-static videos are the dominant GPU load: sampled into up to 64 frames, each re-runs the whole detect→CCIP→SigLIP chain on ~identical content. Add a CPU perceptual-hash frame dedup upstream of the GPU so the redundant frames are never processed at all (not just their embeds). - media.dedupe_frames() + _dhash(): 8×8 difference-hash (64-bit) per frame; greedy keep — a frame survives only if its hash differs from every kept frame by >= min_distance bits (Hamming). A static run collapses to one frame; genuinely distinct scenes all survive. Order + frame_time preserved. - Called in worker._download_decode right after sample_frames, so it runs in the decode stage on the downloader thread (CPU) — the GPU consumers only ever see deduped frames, and buffered video items shrink (less RAM too). - Env-tunable FRAME_DEDUPE_DISTANCE (default 8; higher keeps more frames for brief localized changes an 8×8 hash can miss; 0 disables). Logs `video frames N→M` when it drops any, so video load reduction is visible. Complements the spatial per-frame crop dedup (2026-07-01.2); this is the temporal axis. Build marker 2026-07-01.3. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- agent/fc_agent/app.py | 2 +- agent/fc_agent/config.py | 4 ++++ agent/fc_agent/media.py | 41 ++++++++++++++++++++++++++++++++++++++++ agent/fc_agent/worker.py | 9 +++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index 389a70e..615dab2 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -17,7 +17,7 @@ from .worker import Worker # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-07-01.2 · crop dedup before embed" +VERSION = "2026-07-01.3 · video frame dedup" logbuf.install() cfg = Config.from_env() diff --git a/agent/fc_agent/config.py b/agent/fc_agent/config.py index 5fdf5cc..027cdc1 100644 --- a/agent/fc_agent/config.py +++ b/agent/fc_agent/config.py @@ -38,6 +38,9 @@ class Config: max_regions: int # hard cap on total regions per JOB (submit-size backstop) dedupe_iou: float # crops overlapping >= this (same kind) are near-dupes, # dropped before the embed; >=1.0 disables it + frame_dedupe_distance: int # video frames whose dHash differs by < this many + # bits are near-duplicates, dropped before detect; + # higher keeps more frames, 0 disables @classmethod def from_env(cls) -> Config: @@ -65,4 +68,5 @@ class Config: max_figures=int(os.environ.get("MAX_FIGURES", "8")), max_regions=int(os.environ.get("MAX_REGIONS", "128")), dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")), + frame_dedupe_distance=int(os.environ.get("FRAME_DEDUPE_DISTANCE", "8")), ) diff --git a/agent/fc_agent/media.py b/agent/fc_agent/media.py index 29cbc24..435edfa 100644 --- a/agent/fc_agent/media.py +++ b/agent/fc_agent/media.py @@ -25,6 +25,47 @@ def is_video(mime: str) -> bool: return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"}) +def _dhash(img: Image.Image, size: int = 8) -> int: + """Difference hash: compare adjacent pixels of a (size+1 × size) grayscale + thumbnail → a `size*size`-bit fingerprint. Cheap (64 comparisons on a 72-px + thumbnail) and robust to scaling/compression noise — near-identical frames + hash within a few bits, a real scene change moves many.""" + small = img.convert("L").resize((size + 1, size)) + px = list(small.getdata()) + bits = 0 + for row in range(size): + base = row * (size + 1) + for col in range(size): + bits = (bits << 1) | int(px[base + col] > px[base + col + 1]) + return bits + + +def dedupe_frames( + frames: list[tuple[float, Image.Image]], min_distance: int +) -> list[tuple[float, Image.Image]]: + """Drop visually near-duplicate frames. A near-static video sampled into many + frames re-runs the WHOLE detect→CCIP→SigLIP chain on ~identical frames — the + dominant video load. Greedy perceptual-hash dedup: keep a frame only if its + dHash differs from every already-kept frame by >= min_distance bits (Hamming), + so a static run collapses to one frame while genuinely distinct scenes all + survive. Order + timestamps preserved. CPU-only (64-bit int XORs), so it runs + in the decode stage and spares the GPU the skipped frames entirely. + + min_distance is the coarseness dial: higher keeps more frames (safer for brief + localized changes an 8×8 hash can miss), 0 disables. The first frame is always + kept (nothing to compare against).""" + if min_distance <= 0 or len(frames) <= 1: + return frames + kept: list[tuple[float, Image.Image]] = [] + hashes: list[int] = [] + for t, frame in frames: + h = _dhash(frame) + if all(bin(h ^ k).count("1") >= min_distance for k in hashes): + hashes.append(h) + kept.append((t, frame)) + return kept + + def to_rgb(img: Image.Image) -> Image.Image: """RGB, flattening any transparency onto white first. A naive convert('RGB') on a palette-with-transparency image (common for character PNGs on a clear diff --git a/agent/fc_agent/worker.py b/agent/fc_agent/worker.py index 47c29e8..ee223c5 100644 --- a/agent/fc_agent/worker.py +++ b/agent/fc_agent/worker.py @@ -467,6 +467,15 @@ class Worker: data, job.get("frame_interval_seconds", 4.0), job.get("max_frames", 64), ) or [(None, media.load_image(data))] + # Temporal dedup: a near-static video sampled into many frames re-runs + # the whole detect+embed chain on ~identical frames. Drop near-dup + # frames HERE (decode stage, CPU) so the GPU never sees them. + if len(frames) > 1 and self.cfg.frame_dedupe_distance > 0: + kept = media.dedupe_frames(frames, self.cfg.frame_dedupe_distance) + if len(kept) < len(frames): + log.info("job %s: video frames %d→%d (near-dup dedup)", + job.get("job_id"), len(frames), len(kept)) + frames = kept else: frames = [(None, media.load_image(data))] self._record("decode", time.monotonic() - _t)