diff --git a/alembic/versions/0082_presentation_auto_hide.py b/alembic/versions/0082_presentation_auto_hide.py new file mode 100644 index 0000000..8dc1f3c --- /dev/null +++ b/alembic/versions/0082_presentation_auto_hide.py @@ -0,0 +1,85 @@ +"""presentation-chrome auto-hide (#141) — settings knobs + review table + +MLSettings gains presentation_auto_apply_enabled / _threshold and +presentation_conflict_threshold: banner + editor-screenshot auto-hide on the +sweep with a FLAT threshold (decoupled from content-head graduation), and a +conflict threshold that flags an auto-hide that "also looks like content". + +New table presentation_review records an auto-hidden chrome image that also +scored high on a content head, surfaced in the Hidden view for a keep-hidden / +un-hide decision. Resolved rows are pruned by retention. + +Revision ID: 0082 +Revises: 0081 +Create Date: 2026-07-07 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0082" +down_revision: Union[str, None] = "0081" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "presentation_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("true"), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "presentation_auto_apply_threshold", sa.Float(), nullable=False, + server_default=sa.text("0.90"), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "presentation_conflict_threshold", sa.Float(), nullable=False, + server_default=sa.text("0.50"), + ), + ) + op.create_table( + "presentation_review", + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column( + "conflict_tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("conflict_score", sa.Float(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), + ) + # The review list queries the unresolved flags (resolved_at IS NULL). + op.create_index( + "ix_presentation_review_resolved_at", "presentation_review", + ["resolved_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_presentation_review_resolved_at", table_name="presentation_review" + ) + op.drop_table("presentation_review") + op.drop_column("ml_settings", "presentation_conflict_threshold") + op.drop_column("ml_settings", "presentation_auto_apply_threshold") + op.drop_column("ml_settings", "presentation_auto_apply_enabled") diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 1d290cf..6bf91fe 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -3,9 +3,18 @@ from datetime import UTC, datetime, timedelta from quart import Blueprint, jsonify, request +from sqlalchemy import delete, select, update +from sqlalchemy.orm import aliased from ..extensions import get_session -from ..services.gallery_service import GalleryService +from ..models import ( + ImageRecord, + PresentationReview, + Tag, + TagSuggestionRejection, +) +from ..models.tag import image_tag +from ..services.gallery_service import GalleryService, image_url, thumbnail_url gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") @@ -77,6 +86,9 @@ def _parse_filters(): platform = request.args.get("platform") or None untagged = request.args.get("untagged") in ("1", "true", "yes") no_artist = request.args.get("no_artist") in ("1", "true", "yes") + # Show the presentation chrome (banner / editor screenshot) that the default + # gallery hides — the Hidden view sets this (milestone 141). + include_hidden = request.args.get("include_hidden") in ("1", "true", "yes") date_from = _parse_date(request.args.get("date_from")) date_to = _parse_date(request.args.get("date_to")) if date_to is not None: @@ -88,6 +100,7 @@ def _parse_filters(): "platform": platform, "untagged": untagged, "no_artist": no_artist, "date_from": date_from, "date_to": date_to, + "include_hidden": include_hidden, } return filters, sort @@ -133,7 +146,11 @@ async def similar(): except (KeyError, ValueError): return jsonify({"error": "similar_to query param required"}), 400 # post_id is the exclusive post-detail view — not a similarity scope. - scope = {k: v for k, v in filters.items() if k != "post_id"} + # include_hidden is a gallery-browse flag; similar() has its OWN presentation + # exclusion (a similarity-quality concern, #1274), so drop it here (#141). + scope = { + k: v for k, v in filters.items() if k not in ("post_id", "include_hidden") + } async with get_session() as session: svc = GalleryService(session) try: @@ -211,6 +228,97 @@ async def jump(): return jsonify({"cursor": cursor}) +# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like +# content", surfaced in the gallery's Show-hidden review strip. ----------- +@gallery_bp.route("/hidden-review", methods=["GET"]) +async def hidden_review(): + """Unresolved presentation auto-hide flags, most-concerning first (highest + content score) — for the gallery's Hidden-view review strip.""" + ptag = aliased(Tag) + ctag = aliased(Tag) + async with get_session() as session: + rows = (await session.execute( + select( + PresentationReview.image_record_id, + PresentationReview.tag_id, + PresentationReview.conflict_tag_id, + PresentationReview.conflict_score, + ImageRecord.path, ImageRecord.thumbnail_path, + ImageRecord.sha256, ImageRecord.mime, + ptag.name.label("tag_name"), + ctag.name.label("conflict_name"), + ) + .join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id) + .join(ptag, ptag.id == PresentationReview.tag_id) + .outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id) + .where(PresentationReview.resolved_at.is_(None)) + .order_by(PresentationReview.conflict_score.desc()) + )).all() + return jsonify({"items": [ + { + "image_id": r.image_record_id, + "tag_id": r.tag_id, + "tag_name": r.tag_name, + "conflict_tag_id": r.conflict_tag_id, + "conflict_name": r.conflict_name, + "conflict_score": r.conflict_score, + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), + "image_url": image_url(r.path), + } + for r in rows + ]}) + + +@gallery_bp.route( + "/hidden-review///keep", methods=["POST"] +) +async def hidden_review_keep(image_id, tag_id): + """Keep the auto-hide: resolve the flag; the tag stays applied (#141).""" + async with get_session() as session: + await session.execute( + update(PresentationReview) + .where( + PresentationReview.image_record_id == image_id, + PresentationReview.tag_id == tag_id, + ) + .values(resolved_at=datetime.now(UTC)) + ) + await session.commit() + return "", 204 + + +@gallery_bp.route( + "/hidden-review///unhide", methods=["POST"] +) +async def hidden_review_unhide(image_id, tag_id): + """Un-hide: remove the presentation tag (image returns to the gallery), record + a rejection so the head LEARNS it misfired, and resolve the flag (#141).""" + from sqlalchemy.dialects.postgresql import insert as pg_insert + + async with get_session() as session: + await session.execute( + delete(image_tag).where( + image_tag.c.image_record_id == image_id, + image_tag.c.tag_id == tag_id, + ) + ) + await session.execute( + pg_insert(TagSuggestionRejection) + .values(image_record_id=image_id, tag_id=tag_id) + .on_conflict_do_nothing() + ) + await session.execute( + update(PresentationReview) + .where( + PresentationReview.image_record_id == image_id, + PresentationReview.tag_id == tag_id, + ) + .values(resolved_at=datetime.now(UTC)) + ) + await session.commit() + return "", 204 + + @gallery_bp.route("/image/", methods=["GET"]) async def image_detail(image_id: int): async with get_session() as session: diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index c146b70..52e26fd 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -39,6 +39,9 @@ _EDITABLE = ( "ccip_match_threshold", "ccip_auto_apply_enabled", "ccip_auto_apply_threshold", + "presentation_auto_apply_enabled", + "presentation_auto_apply_threshold", + "presentation_conflict_threshold", "embedder_model_name", "embedder_model_version", *_DETECTOR_FIELDS, @@ -96,6 +99,9 @@ async def get_settings(): "ccip_match_threshold": s.ccip_match_threshold, "ccip_auto_apply_enabled": s.ccip_auto_apply_enabled, "ccip_auto_apply_threshold": s.ccip_auto_apply_threshold, + "presentation_auto_apply_enabled": s.presentation_auto_apply_enabled, + "presentation_auto_apply_threshold": s.presentation_auto_apply_threshold, + "presentation_conflict_threshold": s.presentation_conflict_threshold, "embedder_model_name": s.embedder_model_name, **{f: getattr(s, f) for f in _DETECTOR_FIELDS}, } @@ -150,6 +156,12 @@ def _validate(p: dict) -> str | None: return "ccip_match_threshold must be between 0.5 and 0.999" if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999): return "ccip_auto_apply_threshold must be between 0.5 and 0.999" + # Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is + # consequential); the conflict cut is a plain probability [0,1]. + if not (0.5 <= float(p["presentation_auto_apply_threshold"]) <= 0.999): + return "presentation_auto_apply_threshold must be between 0.5 and 0.999" + if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0): + return "presentation_conflict_threshold must be between 0 and 1" # Embedder model swap (#1190): both must be non-empty. Changing them means a # different embedding space — the operator must re-embed + retrain after. for key in ("embedder_model_name", "embedder_model_version"): diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index ed6e9a8..c8cd723 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -152,6 +152,15 @@ def make_celery() -> Celery: "schedule": 86400.0, # soft auto-apply: drop auto-tags now below # their threshold (m139); no-op unless the auto-apply switch is on }, + "presentation-auto-apply-daily": { + "task": "backend.app.tasks.ml.scheduled_presentation_auto_apply", + "schedule": 86400.0, # auto-hide banner/editor chrome (#141); + # no-op unless presentation_auto_apply_enabled + }, + "prune-presentation-reviews-daily": { + "task": "backend.app.tasks.ml.prune_presentation_reviews", + "schedule": 86400.0, # retention: drop resolved review flags >30d + }, "snapshot-head-metrics-daily": { "task": "backend.app.tasks.maintenance.snapshot_head_metrics", "schedule": 86400.0, diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 0da2555..f06d74d 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -28,6 +28,7 @@ from .pixiv_failed_media import PixivFailedMedia from .pixiv_seen_media import PixivSeenMedia from .post import Post from .post_attachment import PostAttachment +from .presentation_review import PresentationReview from .series_chapter import SeriesChapter from .series_page import SeriesPage from .series_suggestion import SeriesSuggestion @@ -57,6 +58,7 @@ __all__ = [ "SubscribeStarSeenMedia", "Post", "PostAttachment", + "PresentationReview", "SeriesChapter", "SeriesPage", "SeriesSuggestion", diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 80f81f9..0d38e91 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -84,6 +84,22 @@ class MLSettings(Base): # character matches auto-tag. Float, nullable=False, default=0.95 ) + # -- Presentation chrome auto-hide (#141) ------------------------------- + # banner / editor screenshot auto-apply on the sweep with their OWN flat + # threshold (decoupled from content-head graduation). Hiding is consequential + # so it runs HIGH. `wip` is never auto-applied. When an image would be + # auto-hidden but ALSO scores >= presentation_conflict_threshold on a content + # head, it's still hidden but flagged for review (PresentationReview) instead + # of buried silently. ON by default (opt-out); every auto-tag is reversible. + presentation_auto_apply_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + presentation_auto_apply_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.90 + ) + presentation_conflict_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.50 + ) # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069); # existing libraries keep their stored value until the operator re-embeds. embedder_model_version: Mapped[str] = mapped_column( diff --git a/backend/app/models/presentation_review.py b/backend/app/models/presentation_review.py new file mode 100644 index 0000000..d6f91c4 --- /dev/null +++ b/backend/app/models/presentation_review.py @@ -0,0 +1,40 @@ +"""PresentationReview — an auto-hidden presentation tag that ALSO looked like +real content, flagged for operator review (milestone 141). + +When the auto-apply sweep hides an image as chrome (banner / editor screenshot) +but the image ALSO scores highly on a content head, it still hides it but records +this row so the Hidden view can surface it ("⚠ also looks like ") +for a keep-hidden / un-hide decision. Resolved rows are pruned by retention. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Float, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class PresentationReview(Base): + __tablename__ = "presentation_review" + + image_record_id: Mapped[int] = mapped_column( + ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True + ) + # The presentation tag that was auto-applied (banner / editor screenshot). + tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True + ) + # The content tag the image ALSO scored high on — the "concerning" signal. + # SET NULL (not CASCADE): losing the conflict tag shouldn't erase the flag. + conflict_tag_id: Mapped[int | None] = mapped_column( + ForeignKey("tag.id", ondelete="SET NULL"), nullable=True + ) + conflict_score: Mapped[float] = mapped_column(Float, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + # Set when the operator keeps-hidden or un-hides; retention prunes resolved. + resolved_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 94e949f..a05e6c7 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -187,7 +187,7 @@ def _apply_scope( stmt, *, tag_ids, post_id, artist_id, media_type, tag_or_groups=None, tag_exclude=None, platform=None, untagged=False, no_artist=False, - date_from=None, date_to=None, + date_from=None, date_to=None, hidden_tag_ids=None, ): """Apply the composable gallery filters to a statement. @@ -224,6 +224,12 @@ def _apply_scope( stmt = stmt.where(image_in_any_tag_scope(group)) if tag_exclude: stmt = stmt.where(~image_in_any_tag_scope(tag_exclude)) + # Presentation chrome (banner / editor screenshot) is hidden from the default + # gallery — an implicit exclude the caller supplies unless the operator asked + # to include hidden or is explicitly filtering for a presentation tag + # (milestone 141). `wip` is NOT hidden. Resolved to ids by _hidden_tag_ids. + if hidden_tag_ids: + stmt = stmt.where(~image_in_any_tag_scope(hidden_tag_ids)) prov = _provenance_clause(post_id, artist_id) if prov is not None: stmt = stmt.where(prov) @@ -410,6 +416,31 @@ class GalleryService: def __init__(self, session: AsyncSession): self.session = session + async def _hidden_tag_ids( + self, include_hidden, tag_ids, tag_or_groups, + ) -> list[int] | None: + """Presentation-chrome tag ids to implicitly exclude from a gallery query, + or None. None when the caller asked to include hidden, when the operator + is explicitly filtering FOR a presentation tag (they clearly want to see + it), or when no presentation tags exist. (milestone 141)""" + if include_hidden: + return None + rows = await self.session.execute( + select(Tag.id).where( + Tag.is_system.is_(True), + Tag.name.in_(PRESENTATION_SYSTEM_TAGS), + ) + ) + pres = [r[0] for r in rows] + if not pres: + return None + explicit = set(tag_ids or []) + for group in tag_or_groups or []: + explicit.update(group) + if explicit & set(pres): + return None + return pres + async def scroll( self, cursor: str | None, @@ -426,12 +457,16 @@ class GalleryService: no_artist: bool = False, date_from: datetime | None = None, date_to: datetime | None = None, + include_hidden: bool = False, ) -> GalleryPage: if limit < 1 or limit > 200: raise ValueError("limit must be between 1 and 200") _require_single_filter( tag_ids, post_id, artist_id, tag_or_groups, tag_exclude, ) + hidden = await self._hidden_tag_ids( + include_hidden, tag_ids, tag_or_groups, + ) # eff is the ACTIVE sort column (effective_date or earliest_post_date); # the cursor, ordering and year/month grouping all key off it, so the @@ -444,7 +479,7 @@ class GalleryService: artist_id=artist_id, media_type=media_type, tag_or_groups=tag_or_groups, tag_exclude=tag_exclude, platform=platform, untagged=untagged, no_artist=no_artist, - date_from=date_from, date_to=date_to, + date_from=date_from, date_to=date_to, hidden_tag_ids=hidden, ) descending = sort not in _ASCENDING_SORTS @@ -497,6 +532,7 @@ class GalleryService: no_artist: bool = False, date_from: datetime | None = None, date_to: datetime | None = None, + include_hidden: bool = False, ) -> list[TimelineBucket]: eff = _effective_date_col() year_col = func.date_part("year", eff).label("yr") @@ -508,12 +544,15 @@ class GalleryService: _require_single_filter( tag_ids, post_id, artist_id, tag_or_groups, tag_exclude, ) + hidden = await self._hidden_tag_ids( + include_hidden, tag_ids, tag_or_groups, + ) stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, media_type=media_type, tag_or_groups=tag_or_groups, tag_exclude=tag_exclude, platform=platform, untagged=untagged, no_artist=no_artist, - date_from=date_from, date_to=date_to, + date_from=date_from, date_to=date_to, hidden_tag_ids=hidden, ) stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc()) rows = (await self.session.execute(stmt)).all() @@ -527,7 +566,7 @@ class GalleryService: tag_exclude: list[int] | None = None, platform: str | None = None, untagged: bool = False, no_artist: bool = False, date_from: datetime | None = None, - date_to: datetime | None = None, + date_to: datetime | None = None, include_hidden: bool = False, ) -> str | None: """Returns a cursor that, when passed to scroll() with the same sort, positions at the first image of the given year-month. None if the @@ -544,12 +583,15 @@ class GalleryService: _require_single_filter( tag_ids, post_id, artist_id, tag_or_groups, tag_exclude, ) + hidden = await self._hidden_tag_ids( + include_hidden, tag_ids, tag_or_groups, + ) stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, media_type=media_type, tag_or_groups=tag_or_groups, tag_exclude=tag_exclude, platform=platform, untagged=untagged, no_artist=no_artist, - date_from=date_from, date_to=date_to, + date_from=date_from, date_to=date_to, hidden_tag_ids=hidden, ) descending = sort != "oldest" if descending: @@ -574,6 +616,7 @@ class GalleryService: platform: str | None = None, untagged: bool = False, no_artist: bool = False, date_from: datetime | None = None, date_to: datetime | None = None, + include_hidden: bool = False, ) -> GalleryFacets: """Live facet counts scoped to the current filter. Each facet GROUP is computed with all OTHER active filters applied but its OWN selection @@ -584,10 +627,14 @@ class GalleryService: _require_single_filter( tag_ids, post_id, artist_id, tag_or_groups, tag_exclude, ) + hidden = await self._hidden_tag_ids( + include_hidden, tag_ids, tag_or_groups, + ) common = { "tag_ids": tag_ids, "post_id": post_id, "artist_id": artist_id, "media_type": media_type, "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude, + "hidden_tag_ids": hidden, } # total — the full active filter (the headline result count). diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index ae85b46..47e4073 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -32,13 +32,14 @@ from ...models import ( ImageRecord, ImageRegion, MLSettings, + PresentationReview, Tag, TagHead, TagKind, TagPositiveConfirmation, TagSuggestionRejection, ) -from ...models.tag import image_tag +from ...models.tag import PRESENTATION_SYSTEM_TAGS, image_tag from .training_data import ( _AUTO_SOURCES, _auto_apply_point, @@ -649,8 +650,11 @@ def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int: def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int): - """Eligible heads to fire: graduated (auto_apply_threshold set), enough - support, current embedding. Returns the row list (tag_id/name/weights/...).""" + """Eligible CONTENT heads to fire: graduated (auto_apply_threshold set), + enough support, current embedding, NON-system. System tags never auto-apply + via this path — `wip` never auto-applies at all, and banner/editor screenshot + go through the presentation path at their own flat threshold (#141). Returns + the row list (tag_id/name/weights/...).""" return session.execute( select( TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias, @@ -660,6 +664,7 @@ def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int): .where(TagHead.embedding_version == embedding_version) .where(TagHead.auto_apply_threshold.is_not(None)) .where(TagHead.n_pos >= min_pos) + .where(~Tag.is_system) ).all() @@ -743,6 +748,162 @@ def auto_apply_sweep( return {"n_applied": sum(applied), "concepts": concepts} +_PRESENTATION_SOURCE = "presentation_auto" + + +def _presentation_heads(session: Session, embedding_version: str): + """Trained heads for the presentation chrome tags (banner / editor screenshot). + They fire at the FLAT presentation threshold regardless of graduation — a head + exists once the operator has labelled enough chrome (head_min_positives).""" + return session.execute( + select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias) + .join(Tag, Tag.id == TagHead.tag_id) + .where(TagHead.embedding_version == embedding_version) + .where(Tag.is_system.is_(True)) + .where(Tag.name.in_(PRESENTATION_SYSTEM_TAGS)) + ).all() + + +def _conflict_heads(session: Session, embedding_version: str): + """ALL content (non-system) heads — the "does this ALSO look like real + content" signal for the presentation conflict guard (#141).""" + return session.execute( + select(TagHead.tag_id, TagHead.weights, TagHead.bias) + .join(Tag, Tag.id == TagHead.tag_id) + .where(TagHead.embedding_version == embedding_version) + .where(~Tag.is_system) + ).all() + + +def _valued_image_ids(session: Session) -> set[int]: + """Images the operator has shown they value: carrying a HUMAN or CONFIRMED + content (non-system) tag. The presentation sweep never auto-hides these + (guard 1) — you tagged it, so the model doesn't get to bury it (#141).""" + confirmed = exists().where( + TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id, + TagPositiveConfirmation.tag_id == image_tag.c.tag_id, + ) + rows = session.execute( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(~Tag.is_system) + .where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed) + ).all() + return {r[0] for r in rows} + + +def presentation_auto_apply_sweep(session: Session, dry_run: bool = False) -> dict: + """Auto-hide presentation chrome (banner / editor screenshot) at the FLAT + presentation threshold (#141) — NOT the per-head graduated threshold. Two + guards keep it safe: (1) never hide an image carrying a human/confirmed content + tag; (2) if an image about to be hidden ALSO scores >= the conflict threshold + on a content head, still hide it but flag it (PresentationReview) so the Hidden + view surfaces "also looks like " for review. No-op unless + presentation_auto_apply_enabled. numpy-only (no sklearn). Returns + {n_applied, n_flagged, concepts}.""" + import numpy as np + from sqlalchemy.dialects.postgresql import insert as pg_insert + + settings = _settings(session) + if not dry_run and not settings.presentation_auto_apply_enabled: + return {"n_applied": 0, "n_flagged": 0, "concepts": []} + ver = settings.embedder_model_version + pres = _presentation_heads(session, ver) + if not pres: + return {"n_applied": 0, "n_flagged": 0, "concepts": []} + thr = float(settings.presentation_auto_apply_threshold) + conflict_thr = float(settings.presentation_conflict_threshold) + + Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres]) + bp = np.asarray([r.bias for r in pres], dtype=np.float32) + pres_tag_ids = [r.tag_id for r in pres] + pres_names = [r.name for r in pres] + + conf = _conflict_heads(session, ver) + Wc = bc = conf_tag_ids = None + if conf: + Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf]) + bc = np.asarray([r.bias for r in conf], dtype=np.float32) + conf_tag_ids = [r.tag_id for r in conf] + + valued = _valued_image_ids(session) + + # Skip images that already carry, or have rejected, each presentation tag. + skip = {tid: set() for tid in pres_tag_ids} + for tid in pres_tag_ids: + for (iid,) in session.execute( + select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid) + ): + skip[tid].add(iid) + for (iid,) in session.execute( + select(TagSuggestionRejection.image_record_id).where( + TagSuggestionRejection.tag_id == tid + ) + ): + skip[tid].add(iid) + + applied = [0] * len(pres) + n_flagged = 0 + scanned = 0 + all_ids = list(session.execute( + select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None)) + ).scalars()) + for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK): + chunk = all_ids[start:start + _AUTO_APPLY_CHUNK] + emb = _load_embeddings(session, chunk) + cids = [i for i in chunk if i in emb] + if not cids: + continue + Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np) + probs = 1.0 / (1.0 + np.exp(-(Xn @ Wp.T + bp))) # (N, P) + if Wc is not None: + cprobs = 1.0 / (1.0 + np.exp(-(Xn @ Wc.T + bc))) # (N, C) + max_c = cprobs.max(axis=1) + arg_c = cprobs.argmax(axis=1) + scanned += len(cids) + for p in range(len(pres)): + tid = pres_tag_ids[p] + for idx in np.where(probs[:, p] >= thr)[0]: + iid = cids[int(idx)] + if iid in skip[tid] or iid in valued: + continue + skip[tid].add(iid) + applied[p] += 1 + if not dry_run: + session.execute( + pg_insert(image_tag) + .values( + image_record_id=iid, tag_id=tid, + source=_PRESENTATION_SOURCE, + ) + .on_conflict_do_nothing() + ) + # Guard 2: also looks like content → hide but flag for review. + if Wc is not None and float(max_c[idx]) >= conflict_thr: + n_flagged += 1 + if not dry_run: + session.execute( + pg_insert(PresentationReview) + .values( + image_record_id=iid, tag_id=tid, + conflict_tag_id=conf_tag_ids[int(arg_c[idx])], + conflict_score=float(max_c[idx]), + ) + .on_conflict_do_nothing() + ) + if not dry_run: + session.commit() + + concepts = [ + {"tag_id": pres_tag_ids[p], "name": pres_names[p], + "applied": applied[p], "scanned": scanned, "threshold": thr} + for p in range(len(pres)) + ] + return { + "n_applied": sum(applied), "n_flagged": n_flagged, "concepts": concepts, + } + + def retract_auto_applied_heads(session: Session) -> int: """Soft auto-apply (milestone 139): re-score every standing source='head_auto' tag against its CURRENT head and REMOVE the ones now BELOW the head's diff --git a/backend/app/services/ml/training_data.py b/backend/app/services/ml/training_data.py index 6c9802a..8fcf8cc 100644 --- a/backend/app/services/ml/training_data.py +++ b/backend/app/services/ml/training_data.py @@ -29,7 +29,7 @@ from ...models.tag import image_tag # a CCIP reference) unless the operator confirms them (milestone 139). Keeping # auto-applied predictions out of training is what makes them "soft" — a misfire # can't reinforce itself, so the retraction sweep can actually drop it. -_AUTO_SOURCES = ("head_auto", "ccip_auto", "ml_auto") +_AUTO_SOURCES = ("head_auto", "ccip_auto", "ml_auto", "presentation_auto") def _hygiene_excluded_ids(session: Session) -> set[int]: diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index 2172e71..0f6343a 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -594,6 +594,46 @@ def scheduled_ccip_auto_apply() -> str: return f"applied={applied}" +@celery.task( + name="backend.app.tasks.ml.scheduled_presentation_auto_apply", + soft_time_limit=1800, time_limit=2100, +) +def scheduled_presentation_auto_apply() -> str: + """Auto-hide presentation chrome (banner / editor screenshot) on a daily + passive sweep (#141). No-op unless presentation_auto_apply_enabled. Idempotent + — already-hidden images are skipped — so an interrupted run simply re-runs next + cycle (that IS the recovery). Wall-clock bounded by the task time limits.""" + from ..services.ml.heads import presentation_auto_apply_sweep + + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + result = presentation_auto_apply_sweep(session) + return f"applied={result['n_applied']} flagged={result['n_flagged']}" + + +@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews") +def prune_presentation_reviews() -> str: + """Retention (rule 89): drop RESOLVED presentation-review flags older than 30 + days — the operator has acted on them, so they're just history (#141).""" + from datetime import UTC, datetime, timedelta + + from sqlalchemy import delete as sa_delete + + from ..models import PresentationReview + + cutoff = datetime.now(UTC) - timedelta(days=30) + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + res = session.execute( + sa_delete(PresentationReview).where( + PresentationReview.resolved_at.is_not(None), + PresentationReview.resolved_at < cutoff, + ) + ) + session.commit() + return f"pruned={res.rowcount}" + + @celery.task( name="backend.app.tasks.ml.scheduled_retract_auto_tags", soft_time_limit=1800, time_limit=2100, diff --git a/frontend/src/components/gallery/GalleryFacetPanel.vue b/frontend/src/components/gallery/GalleryFacetPanel.vue index 94a6d02..a27bf24 100644 --- a/frontend/src/components/gallery/GalleryFacetPanel.vue +++ b/frontend/src/components/gallery/GalleryFacetPanel.vue @@ -34,6 +34,15 @@ :color="store.filter.no_artist ? 'accent' : undefined" @click="toggleFlag('no_artist')" >No artist{{ facetCount('no_artist') }} + + Show hidden diff --git a/frontend/src/components/gallery/GalleryFilterBar.vue b/frontend/src/components/gallery/GalleryFilterBar.vue index f0a9b7e..779a4da 100644 --- a/frontend/src/components/gallery/GalleryFilterBar.vue +++ b/frontend/src/components/gallery/GalleryFilterBar.vue @@ -160,7 +160,7 @@ let debounce = null const refineCount = computed(() => { const f = store.filter return (f.platform ? 1 : 0) + (f.untagged ? 1 : 0) + (f.no_artist ? 1 : 0) - + (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0) + + (f.include_hidden ? 1 : 0) + (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0) }) const hasRefineFilters = computed(() => refineCount.value > 0) diff --git a/frontend/src/components/gallery/HiddenReviewStrip.vue b/frontend/src/components/gallery/HiddenReviewStrip.vue new file mode 100644 index 0000000..7c25e28 --- /dev/null +++ b/frontend/src/components/gallery/HiddenReviewStrip.vue @@ -0,0 +1,151 @@ + + + + + diff --git a/frontend/src/components/modal/TagChip.vue b/frontend/src/components/modal/TagChip.vue index 7cf2f59..ccd698d 100644 --- a/frontend/src/components/modal/TagChip.vue +++ b/frontend/src/components/modal/TagChip.vue @@ -10,7 +10,7 @@ @click:close="$emit('remove', tag.id)" > {{ iconFor(tag.kind) }} - {{ tag.name }}{{ tag.name }}mdi-shield-outlinemdi-check + >mdi-check +
+
+ mdi-image-off-outline + Hide presentation chrome + +
+

+ Auto-hide banners and editor screenshots from the gallery once a head has + learned them (≥ {{ minPositives }} examples) and clears + {{ Math.round((presentationThresholdInput || 0) * 100) }}% confidence. + wip is never auto-hidden. If a hidden image also looks like + real content (≥ {{ Math.round((presentationConflictInput || 0) * 100) }}% + on a content tag), it's flagged in the Hidden view instead of buried. + Every auto-hide is reversible. +

+
+ + +
+
+
How auto-apply is landing
@@ -218,6 +254,11 @@ const autoStatus = ref(null) const metricsData = ref(null) let autoTimer = null +// --- Presentation chrome auto-hide state (#141) --- +const presentationEnabled = ref(true) +const presentationThresholdInput = ref(0.90) +const presentationConflictInput = ref(0.50) + const autoRunning = computed(() => autoStatus.value?.running_id != null) const lastSweep = computed(() => (autoStatus.value?.runs || []).find(r => r.status !== 'running') || null) @@ -248,6 +289,9 @@ onMounted(async () => { autoEnabled.value = !!s.head_auto_apply_enabled autoPrecisionInput.value = s.head_auto_apply_precision ?? 0.97 autoMinPosInput.value = s.head_auto_apply_min_positives ?? 30 + presentationEnabled.value = s.presentation_auto_apply_enabled ?? true + presentationThresholdInput.value = s.presentation_auto_apply_threshold ?? 0.90 + presentationConflictInput.value = s.presentation_conflict_threshold ?? 0.50 } catch { /* non-fatal */ } await refresh() if (running.value) startPoll() @@ -332,6 +376,32 @@ async function onSaveSettings() { settingBusy.value = false } } + +async function onTogglePresentation(val) { + settingBusy.value = true + try { + await mlSettings.patchSettings({ presentation_auto_apply_enabled: !!val }) + toast({ text: val ? 'Chrome auto-hide on' : 'Chrome auto-hide off', type: 'success' }) + } catch (e) { + presentationEnabled.value = !val // revert the switch + toast({ text: `Could not update: ${e.message}`, type: 'error' }) + } finally { + settingBusy.value = false + } +} +async function onSavePresentation() { + settingBusy.value = true + try { + await mlSettings.patchSettings({ + presentation_auto_apply_threshold: Number(presentationThresholdInput.value), + presentation_conflict_threshold: Number(presentationConflictInput.value), + }) + } catch (e) { + toast({ text: `Could not save: ${e.message}`, type: 'error' }) + } finally { + settingBusy.value = false + } +} function onPreview() { startSweep(true) } function onApplyNow() { startSweep(false) } async function startSweep(dryRun) { diff --git a/frontend/src/stores/gallery.js b/frontend/src/stores/gallery.js index 617cc73..334c3bd 100644 --- a/frontend/src/stores/gallery.js +++ b/frontend/src/stores/gallery.js @@ -30,6 +30,9 @@ export const useGalleryStore = defineStore('gallery', () => { tag_or: [], tag_exclude: [], // Phase-2 faceted refine params. platform: null, untagged: false, no_artist: false, + // Reveal the presentation chrome (banner / editor screenshot) the gallery + // hides by default (milestone 141). + include_hidden: false, date_from: null, date_to: null, // Phase-3 visual similarity: when set, the gallery is in "similar mode" — // ranked by cosine distance to this image, bounded top-N, no cursor. @@ -158,6 +161,7 @@ export const useGalleryStore = defineStore('gallery', () => { if (filter.value.platform) p.platform = filter.value.platform if (filter.value.untagged) p.untagged = '1' if (filter.value.no_artist) p.no_artist = '1' + if (filter.value.include_hidden) p.include_hidden = '1' if (filter.value.date_from) p.date_from = filter.value.date_from if (filter.value.date_to) p.date_to = filter.value.date_to return p @@ -196,6 +200,7 @@ export const useGalleryStore = defineStore('gallery', () => { platform: q.platform || null, untagged: _truthy(q.untagged), no_artist: _truthy(q.no_artist), + include_hidden: _truthy(q.include_hidden), date_from: _parseDate(q.date_from), date_to: _parseDate(q.date_to), similar_to: _toId(q.similar_to), @@ -284,7 +289,8 @@ export function cloneFilter(f) { tag_exclude: [...(f.tag_exclude || [])], artist_id: f.artist_id, media_type: f.media_type, sort: f.sort, platform: f.platform, untagged: f.untagged, - no_artist: f.no_artist, date_from: f.date_from, date_to: f.date_to, + no_artist: f.no_artist, include_hidden: f.include_hidden, + date_from: f.date_from, date_to: f.date_to, similar_to: f.similar_to, } } @@ -301,6 +307,7 @@ export function filterToQuery(f) { if (f.platform) q.platform = f.platform if (f.untagged) q.untagged = '1' if (f.no_artist) q.no_artist = '1' + if (f.include_hidden) q.include_hidden = '1' if (f.date_from) q.date_from = f.date_from if (f.date_to) q.date_to = f.date_to if (f.similar_to) q.similar_to = String(f.similar_to) diff --git a/frontend/src/views/GalleryView.vue b/frontend/src/views/GalleryView.vue index 0358a77..be35891 100644 --- a/frontend/src/views/GalleryView.vue +++ b/frontend/src/views/GalleryView.vue @@ -13,6 +13,7 @@
+
@@ -33,6 +34,7 @@ import { useGalleryStore } from '../stores/gallery.js' import { useModalStore } from '../stores/modal.js' import GalleryGrid from '../components/gallery/GalleryGrid.vue' import GalleryFilterBar from '../components/gallery/GalleryFilterBar.vue' +import HiddenReviewStrip from '../components/gallery/HiddenReviewStrip.vue' import TimelineSidebar from '../components/gallery/TimelineSidebar.vue' import EmptyState from '../components/gallery/EmptyState.vue' import PostInfoHeader from '../components/gallery/PostInfoHeader.vue' diff --git a/tests/test_api_ml_admin.py b/tests/test_api_ml_admin.py index 33faf09..df9141d 100644 --- a/tests/test_api_ml_admin.py +++ b/tests/test_api_ml_admin.py @@ -81,6 +81,35 @@ async def test_detector_settings_defaults_patch_and_validation(client): "/api/ml/settings", json={"detector_max_regions": 0})).status_code == 400 +@pytest.mark.asyncio +async def test_presentation_settings_defaults_patch_and_validation(client): + # #141: presentation-chrome auto-hide knobs (banner/editor auto-apply + + # the "also looks like content" conflict threshold) are exposed + editable. + body = await (await client.get("/api/ml/settings")).get_json() + assert body["presentation_auto_apply_enabled"] is True + assert body["presentation_auto_apply_threshold"] == 0.90 + assert body["presentation_conflict_threshold"] == 0.50 + + ok = await client.patch("/api/ml/settings", json={ + "presentation_auto_apply_enabled": False, + "presentation_auto_apply_threshold": 0.95, + "presentation_conflict_threshold": 0.4, + }) + assert ok.status_code == 200 + out = await ok.get_json() + assert out["presentation_auto_apply_enabled"] is False + assert out["presentation_auto_apply_threshold"] == 0.95 + assert out["presentation_conflict_threshold"] == 0.4 + + # Auto-apply threshold below the floor + conflict cut above 1 are rejected. + assert (await client.patch( + "/api/ml/settings", + json={"presentation_auto_apply_threshold": 0.4})).status_code == 400 + assert (await client.patch( + "/api/ml/settings", + json={"presentation_conflict_threshold": 1.5})).status_code == 400 + + @pytest.mark.asyncio async def test_embedder_models_list(client): # #1203: the dropdown reads the supported-model list from the server. diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index 9c4a54b..96266cc 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta import pytest +from sqlalchemy import select from backend.app.models import ImageRecord, Tag, TagKind from backend.app.models.tag import image_tag @@ -73,6 +74,47 @@ async def test_scroll_returns_newest_first(db): assert page.images[0].created_at > page.images[-1].created_at +async def _system_tag(db, name): + return (await db.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == name) + )).scalar_one() + + +@pytest.mark.asyncio +async def test_scroll_hides_presentation_chrome_by_default(db): + # banner / editor screenshot (presentation system tags) are hidden from the + # default gallery; wip (also a system tag) is NOT — it's real, in-progress + # art (milestone 141). Seeded system tags survive the harness TRUNCATE. + imgs = await _seed_images(db, 3, sha_prefix="p") + banner = await _system_tag(db, "banner") + wip = await _system_tag(db, "wip") + await db.execute(image_tag.insert().values( + image_record_id=imgs[0].id, tag_id=banner.id, source="manual")) + await db.execute(image_tag.insert().values( + image_record_id=imgs[1].id, tag_id=wip.id, source="manual")) + await db.flush() + svc = GalleryService(db) + + # Default: the banner image is hidden; the wip image + the plain image stay. + default_ids = {i.id for i in (await svc.scroll(cursor=None, limit=10)).images} + assert imgs[0].id not in default_ids # banner hidden + assert imgs[1].id in default_ids # wip visible + assert imgs[2].id in default_ids # plain visible + + # include_hidden surfaces the banner image (the Hidden view). + shown = {i.id for i in ( + await svc.scroll(cursor=None, limit=10, include_hidden=True) + ).images} + assert imgs[0].id in shown + + # Filtering explicitly FOR banner shows it (a view exclusively for a + # presentation tag suppresses the implicit hide). + only_banner = {i.id for i in ( + await svc.scroll(cursor=None, limit=10, tag_ids=[banner.id]) + ).images} + assert only_banner == {imgs[0].id} + + @pytest.mark.asyncio async def test_scroll_sorts_by_earliest_post_date(db): """posted_new/posted_old key off earliest_post_date (original publish across diff --git a/tests/test_hidden_review.py b/tests/test_hidden_review.py new file mode 100644 index 0000000..5e15359 --- /dev/null +++ b/tests/test_hidden_review.py @@ -0,0 +1,89 @@ +"""Hidden-view review endpoints (#141): list flagged auto-hides + keep / un-hide.""" +import pytest +from sqlalchemy import select + +from backend.app.models import ( + ImageRecord, + PresentationReview, + Tag, + TagKind, + TagSuggestionRejection, +) +from backend.app.models.tag import image_tag + +pytestmark = pytest.mark.integration + + +async def _setup_flag(db): + banner = (await db.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == "banner") + )).scalar_one() + content = Tag(name="looksreal", kind=TagKind.general) + db.add(content) + img = ImageRecord( + path="/images/rv.jpg", sha256="rv" * 32, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", + integrity_status="unknown", + ) + db.add(img) + await db.flush() + await db.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=banner.id, source="presentation_auto")) + db.add(PresentationReview( + image_record_id=img.id, tag_id=banner.id, + conflict_tag_id=content.id, conflict_score=0.8, + )) + await db.commit() + return img, banner, content + + +async def _source(db, image_id, tag_id): + return (await db.execute( + select(image_tag.c.source).where( + image_tag.c.image_record_id == image_id, + image_tag.c.tag_id == tag_id, + ) + )).scalar_one_or_none() + + +@pytest.mark.asyncio +async def test_hidden_review_lists_flag(client, db): + img, banner, content = await _setup_flag(db) + body = await (await client.get("/api/gallery/hidden-review")).get_json() + assert len(body["items"]) == 1 + it = body["items"][0] + assert it["image_id"] == img.id + assert it["tag_id"] == banner.id + assert it["conflict_tag_id"] == content.id + assert it["conflict_name"] == "looksreal" + assert it["conflict_score"] == 0.8 + assert it["thumbnail_url"] + + +@pytest.mark.asyncio +async def test_hidden_review_keep_resolves_but_keeps_tag(client, db): + img, banner, _ = await _setup_flag(db) + resp = await client.post( + f"/api/gallery/hidden-review/{img.id}/{banner.id}/keep") + assert resp.status_code == 204 + body = await (await client.get("/api/gallery/hidden-review")).get_json() + assert body["items"] == [] # flag resolved + assert await _source(db, img.id, banner.id) == "presentation_auto" # tag stays + + +@pytest.mark.asyncio +async def test_hidden_review_unhide_removes_tag_and_records_rejection(client, db): + img, banner, _ = await _setup_flag(db) + resp = await client.post( + f"/api/gallery/hidden-review/{img.id}/{banner.id}/unhide") + assert resp.status_code == 204 + assert await _source(db, img.id, banner.id) is None # back in the gallery + rej = (await db.execute( + select(TagSuggestionRejection).where( + TagSuggestionRejection.image_record_id == img.id, + TagSuggestionRejection.tag_id == banner.id, + ) + )).scalar_one_or_none() + assert rej is not None # head learns it misfired + body = await (await client.get("/api/gallery/hidden-review")).get_json() + assert body["items"] == [] diff --git a/tests/test_presentation_auto_apply.py b/tests/test_presentation_auto_apply.py new file mode 100644 index 0000000..ac7a4d7 --- /dev/null +++ b/tests/test_presentation_auto_apply.py @@ -0,0 +1,152 @@ +"""Presentation-chrome auto-hide sweep (#141). numpy-only (no sklearn), so the +apply + guard logic is tested directly via the sync session.""" +import pytest +from sqlalchemy import select + +from backend.app.models import ( + HeadAutoApplyRun, + ImageRecord, + MLSettings, + PresentationReview, + Tag, + TagHead, + TagKind, +) +from backend.app.models.tag import image_tag +from backend.app.services.ml.heads import ( + auto_apply_sweep, + presentation_auto_apply_sweep, +) + +pytestmark = pytest.mark.integration + + +def _emb(slot: int) -> list[float]: + v = [0.0] * 1152 + v[slot] = 3.0 + return v + + +def _img(db, sha: str, emb) -> ImageRecord: + img = ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", + integrity_status="unknown", siglip_embedding=emb, + ) + db.add(img) + db.flush() + return img + + +def _head(db, tag_id: int, slot: int, *, weight=1.0): + # weight 3.0 → score sigmoid(3)=0.95 clears the 0.90 presentation floor; + # weight 1.0 → sigmoid(1)=0.73 clears the 0.50 conflict floor. + s = db.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one() + w = [0.0] * 1152 + w[slot] = weight + db.add(TagHead( + tag_id=tag_id, embedding_version=s.embedder_model_version, + weights=w, bias=0.0, suggest_threshold=0.5, auto_apply_threshold=0.5, + n_pos=60, n_neg=90, ap=0.9, precision_cv=0.98, recall=0.7, + )) + + +def _system_tag(db, name): + return db.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == name) + ).scalar_one() + + +def _source(db, image_id, tag_id): + return db.execute( + select(image_tag.c.source) + .where(image_tag.c.image_record_id == image_id) + .where(image_tag.c.tag_id == tag_id) + ).scalar_one_or_none() + + +def test_presentation_sweep_hides_chrome(db_sync): + banner = _system_tag(db_sync, "banner") + _head(db_sync, banner.id, 0, weight=3.0) + img = _img(db_sync, "a" * 64, _emb(0)) + db_sync.commit() + res = presentation_auto_apply_sweep(db_sync) + assert res["n_applied"] == 1 + assert _source(db_sync, img.id, banner.id) == "presentation_auto" + + +def test_presentation_sweep_hard_skips_valued_image(db_sync): + # An image the operator already tagged with a content tag is never auto-hidden. + banner = _system_tag(db_sync, "banner") + _head(db_sync, banner.id, 0, weight=3.0) + content = Tag(name="mychar", kind=TagKind.character) + db_sync.add(content) + db_sync.flush() + img = _img(db_sync, "b" * 64, _emb(0)) + db_sync.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=content.id, source="manual")) + db_sync.commit() + res = presentation_auto_apply_sweep(db_sync) + assert res["n_applied"] == 0 + assert _source(db_sync, img.id, banner.id) is None + + +def test_presentation_sweep_flags_conflict(db_sync): + # Matches banner AND scores high on a content head → hidden but flagged with + # that content tag as the conflict. + banner = _system_tag(db_sync, "banner") + _head(db_sync, banner.id, 0, weight=3.0) + content = Tag(name="looksreal", kind=TagKind.general) + db_sync.add(content) + db_sync.flush() + _head(db_sync, content.id, 0, weight=1.0) # content head also fires + img = _img(db_sync, "c" * 64, _emb(0)) + db_sync.commit() + res = presentation_auto_apply_sweep(db_sync) + assert res["n_applied"] == 1 + assert res["n_flagged"] == 1 + assert _source(db_sync, img.id, banner.id) == "presentation_auto" + flag = db_sync.execute( + select(PresentationReview).where( + PresentationReview.image_record_id == img.id, + PresentationReview.tag_id == banner.id, + ) + ).scalar_one() + assert flag.conflict_tag_id == content.id + assert flag.conflict_score >= 0.5 + + +def test_presentation_sweep_disabled_is_noop(db_sync): + s = db_sync.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one() + s.presentation_auto_apply_enabled = False + banner = _system_tag(db_sync, "banner") + _head(db_sync, banner.id, 0, weight=3.0) + img = _img(db_sync, "d" * 64, _emb(0)) + db_sync.commit() + res = presentation_auto_apply_sweep(db_sync) + assert res["n_applied"] == 0 + assert _source(db_sync, img.id, banner.id) is None + + +def test_presentation_sweep_ignores_wip(db_sync): + # wip is a system tag but NOT presentation chrome → never auto-applied. + wip = _system_tag(db_sync, "wip") + _head(db_sync, wip.id, 0, weight=3.0) + img = _img(db_sync, "e" * 64, _emb(0)) + db_sync.commit() + res = presentation_auto_apply_sweep(db_sync) + assert res["n_applied"] == 0 + assert _source(db_sync, img.id, wip.id) is None + + +def test_content_sweep_never_fires_system_tags(db_sync): + # A graduated banner (system) head must NOT auto-apply via the content path. + banner = _system_tag(db_sync, "banner") + _head(db_sync, banner.id, 0, weight=3.0) + img = _img(db_sync, "f" * 64, _emb(0)) + run = HeadAutoApplyRun(dry_run=False, params={}, status="running") + db_sync.add(run) + db_sync.flush() + db_sync.commit() + auto_apply_sweep(db_sync, run, dry_run=False) + assert _source(db_sync, img.id, banner.id) is None diff --git a/tests/test_presentation_scheduled.py b/tests/test_presentation_scheduled.py new file mode 100644 index 0000000..7272b16 --- /dev/null +++ b/tests/test_presentation_scheduled.py @@ -0,0 +1,98 @@ +"""Scheduled presentation tasks (#141): the daily auto-hide sweep wrapper + +resolved-flag retention. The task opens its own session via +_sync_session_factory, monkeypatched here to run against the test's db_sync.""" +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select + +from backend.app.models import ( + ImageRecord, + MLSettings, + PresentationReview, + Tag, + TagHead, +) +from backend.app.tasks.ml import ( + prune_presentation_reviews, + scheduled_presentation_auto_apply, +) + +pytestmark = pytest.mark.integration + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _img(db, sha, emb=None): + img = ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha * 32, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", + integrity_status="unknown", siglip_embedding=emb, + ) + db.add(img) + db.flush() + return img + + +def test_scheduled_presentation_sweep_hides_chrome(db_sync, monkeypatch): + monkeypatch.setattr( + "backend.app.tasks.ml._sync_session_factory", lambda: _sf(db_sync) + ) + banner = db_sync.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == "banner") + ).scalar_one() + s = db_sync.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one() + w = [0.0] * 1152 + w[0] = 3.0 + db_sync.add(TagHead( + tag_id=banner.id, embedding_version=s.embedder_model_version, + weights=w, bias=0.0, suggest_threshold=0.5, auto_apply_threshold=0.5, + n_pos=60, n_neg=90, ap=0.9, precision_cv=0.98, recall=0.7, + )) + emb = [0.0] * 1152 + emb[0] = 3.0 + _img(db_sync, "sc", emb) + db_sync.commit() + assert "applied=1" in scheduled_presentation_auto_apply() + + +def test_prune_presentation_reviews_drops_old_resolved(db_sync, monkeypatch): + monkeypatch.setattr( + "backend.app.tasks.ml._sync_session_factory", lambda: _sf(db_sync) + ) + banner = db_sync.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == "banner") + ).scalar_one() + a, b, c = _img(db_sync, "p1"), _img(db_sync, "p2"), _img(db_sync, "p3") + old = datetime.now(UTC) - timedelta(days=40) + recent = datetime.now(UTC) - timedelta(days=1) + # old resolved → pruned; recent resolved → kept; unresolved → kept. + db_sync.add(PresentationReview( + image_record_id=a.id, tag_id=banner.id, conflict_score=0.7, resolved_at=old)) + db_sync.add(PresentationReview( + image_record_id=b.id, tag_id=banner.id, conflict_score=0.7, resolved_at=recent)) + db_sync.add(PresentationReview( + image_record_id=c.id, tag_id=banner.id, conflict_score=0.7)) + db_sync.commit() + assert prune_presentation_reviews() == "pruned=1" + remaining = db_sync.execute( + select(PresentationReview.image_record_id) + ).scalars().all() + assert set(remaining) == {b.id, c.id}