Merge pull request 'Ship: system-tag refactor + soft WIP tier + Explore reach (#1464, #1474, #1476)' (#223) from dev into main
Build images / sign-extension (push) Successful in 3s
CI / lint (push) Successful in 4s
Build images / build-agent (push) Successful in 9s
Build images / build-ml (push) Successful in 9s
Build images / build-web (push) Successful in 9s
CI / frontend-build (push) Successful in 22s
CI / backend-lint-and-test (push) Successful in 34s
CI / integration (push) Successful in 3m45s

This commit was merged in pull request #223.
This commit is contained in:
2026-07-13 12:13:19 -04:00
29 changed files with 994 additions and 151 deletions
@@ -0,0 +1,61 @@
"""process auto-apply settings + review mode (#1464) — system-tag refactor
The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS
group) their own provisional auto-apply, parallel to the presentation (chrome)
sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library
auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict
threshold. presentation_review gains a `mode` column so one review surface serves
both chrome and process flags (existing rows backfill 'chrome'). server_defaults
so the existing rows fill cleanly.
Revision ID: 0086
Revises: 0085
Create Date: 2026-07-13
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0086"
down_revision: Union[str, None] = "0085"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"ml_settings",
sa.Column(
"process_auto_apply_enabled", sa.Boolean(), nullable=False,
server_default=sa.text("false"),
),
)
op.add_column(
"ml_settings",
sa.Column(
"process_auto_apply_threshold", sa.Float(), nullable=False,
server_default="0.90",
),
)
op.add_column(
"ml_settings",
sa.Column(
"process_conflict_threshold", sa.Float(), nullable=False,
server_default="0.50",
),
)
op.add_column(
"presentation_review",
sa.Column(
"mode", sa.String(16), nullable=False,
server_default="chrome",
),
)
def downgrade() -> None:
op.drop_column("presentation_review", "mode")
op.drop_column("ml_settings", "process_conflict_threshold")
op.drop_column("ml_settings", "process_auto_apply_threshold")
op.drop_column("ml_settings", "process_auto_apply_enabled")
@@ -0,0 +1,33 @@
"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled
The soft tier also tags sketch/doodle/scribble titles, but with a provisional source
that never trains the head. OFF by default (a lower-precision tier is opt-in).
server_default so the existing singleton row (id=1) fills cleanly.
Revision ID: 0087
Revises: 0086
Create Date: 2026-07-13
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0087"
down_revision: Union[str, None] = "0086"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"import_settings",
sa.Column(
"wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False,
server_default=sa.text("false"),
),
)
def downgrade() -> None:
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
+19 -3
View File
@@ -148,6 +148,17 @@ async def similar():
# Explore passes exclude_wip=1 to also drop work-in-progress from the # Explore passes exclude_wip=1 to also drop work-in-progress from the
# rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274). # rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True") exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
# Explore reach (#1476): 0 = nearest (gallery default), →1 reaches into farther
# distance bands so the walk can escape a dense cluster. exclude_ids = the
# breadcrumb, so already-walked images aren't re-served as neighbours.
try:
reach = max(0.0, min(1.0, float(request.args.get("reach", "0"))))
except ValueError:
reach = 0.0
exclude_ids = [
int(x) for x in request.args.get("exclude_ids", "").split(",")
if x.strip().isdigit()
] or None
# post_id is the exclusive post-detail view — not a similarity scope. # post_id is the exclusive post-detail view — not a similarity scope.
# include_hidden is a gallery-browse flag; similar() has its OWN presentation # include_hidden is a gallery-browse flag; similar() has its OWN presentation
# exclusion (a similarity-quality concern, #1274), so drop it here (#141). # exclusion (a similarity-quality concern, #1274), so drop it here (#141).
@@ -158,7 +169,8 @@ async def similar():
svc = GalleryService(session) svc = GalleryService(session)
try: try:
images = await svc.similar( images = await svc.similar(
image_id=similar_to, limit=limit, exclude_wip=exclude_wip, **scope) image_id=similar_to, limit=limit, exclude_wip=exclude_wip,
reach=reach, exclude_ids=exclude_ids, **scope)
except ValueError as exc: except ValueError as exc:
return jsonify({"error": str(exc)}), 400 return jsonify({"error": str(exc)}), 400
if images is None: if images is None:
@@ -236,8 +248,10 @@ async def jump():
# content", surfaced in the gallery's Show-hidden review strip. ----------- # content", surfaced in the gallery's Show-hidden review strip. -----------
@gallery_bp.route("/hidden-review", methods=["GET"]) @gallery_bp.route("/hidden-review", methods=["GET"])
async def hidden_review(): async def hidden_review():
"""Unresolved presentation auto-hide flags, most-concerning first (highest """Unresolved system-tag auto-apply review flags (chrome + process, #1464),
content score) — for the gallery's Hidden-view review strip.""" most-concerning first (highest content score) — for the review strip. `mode`
tells the client whether the flagged tag hid the image ('chrome') or left it
visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
ptag = aliased(Tag) ptag = aliased(Tag)
ctag = aliased(Tag) ctag = aliased(Tag)
async with get_session() as session: async with get_session() as session:
@@ -247,6 +261,7 @@ async def hidden_review():
PresentationReview.tag_id, PresentationReview.tag_id,
PresentationReview.conflict_tag_id, PresentationReview.conflict_tag_id,
PresentationReview.conflict_score, PresentationReview.conflict_score,
PresentationReview.mode,
ImageRecord.path, ImageRecord.thumbnail_path, ImageRecord.path, ImageRecord.thumbnail_path,
ImageRecord.sha256, ImageRecord.mime, ImageRecord.sha256, ImageRecord.mime,
ptag.name.label("tag_name"), ptag.name.label("tag_name"),
@@ -266,6 +281,7 @@ async def hidden_review():
"conflict_tag_id": r.conflict_tag_id, "conflict_tag_id": r.conflict_tag_id,
"conflict_name": r.conflict_name, "conflict_name": r.conflict_name,
"conflict_score": r.conflict_score, "conflict_score": r.conflict_score,
"mode": r.mode,
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
"image_url": image_url(r.path), "image_url": image_url(r.path),
} }
+12
View File
@@ -42,6 +42,9 @@ _EDITABLE = (
"presentation_auto_apply_enabled", "presentation_auto_apply_enabled",
"presentation_auto_apply_threshold", "presentation_auto_apply_threshold",
"presentation_conflict_threshold", "presentation_conflict_threshold",
"process_auto_apply_enabled",
"process_auto_apply_threshold",
"process_conflict_threshold",
"embedder_model_name", "embedder_model_name",
"embedder_model_version", "embedder_model_version",
*_DETECTOR_FIELDS, *_DETECTOR_FIELDS,
@@ -102,6 +105,9 @@ async def get_settings():
"presentation_auto_apply_enabled": s.presentation_auto_apply_enabled, "presentation_auto_apply_enabled": s.presentation_auto_apply_enabled,
"presentation_auto_apply_threshold": s.presentation_auto_apply_threshold, "presentation_auto_apply_threshold": s.presentation_auto_apply_threshold,
"presentation_conflict_threshold": s.presentation_conflict_threshold, "presentation_conflict_threshold": s.presentation_conflict_threshold,
"process_auto_apply_enabled": s.process_auto_apply_enabled,
"process_auto_apply_threshold": s.process_auto_apply_threshold,
"process_conflict_threshold": s.process_conflict_threshold,
"embedder_model_name": s.embedder_model_name, "embedder_model_name": s.embedder_model_name,
**{f: getattr(s, f) for f in _DETECTOR_FIELDS}, **{f: getattr(s, f) for f in _DETECTOR_FIELDS},
} }
@@ -162,6 +168,12 @@ def _validate(p: dict) -> str | None:
return "presentation_auto_apply_threshold must be between 0.5 and 0.999" return "presentation_auto_apply_threshold must be between 0.5 and 0.999"
if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0): if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
return "presentation_conflict_threshold must be between 0 and 1" return "presentation_conflict_threshold must be between 0 and 1"
# Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
# low-harm (excludes-from-training + a review flag), but keep the same bar.
if not (0.5 <= float(p["process_auto_apply_threshold"]) <= 0.999):
return "process_auto_apply_threshold must be between 0.5 and 0.999"
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
return "process_conflict_threshold must be between 0 and 1"
# Embedder model swap (#1190): both must be non-empty. Changing them means a # Embedder model swap (#1190): both must be non-empty. Changing them means a
# different embedding space — the operator must re-embed + retrain after. # different embedding space — the operator must re-embed + retrain after.
for key in ("embedder_model_name", "embedder_model_version"): for key in ("embedder_model_name", "embedder_model_version"):
+8
View File
@@ -49,6 +49,7 @@ _EDITABLE_FIELDS = (
"translation_target_lang", "translation_target_lang",
"translation_min_confidence", "translation_min_confidence",
"wip_title_tagging_enabled", "wip_title_tagging_enabled",
"wip_soft_title_tagging_enabled",
) )
# Per-host external-download toggles — all plain booleans, validated uniformly. # Per-host external-download toggles — all plain booleans, validated uniformly.
@@ -91,6 +92,7 @@ async def get_import_settings():
"translation_target_lang": row.translation_target_lang, "translation_target_lang": row.translation_target_lang,
"translation_min_confidence": row.translation_min_confidence, "translation_min_confidence": row.translation_min_confidence,
"wip_title_tagging_enabled": row.wip_title_tagging_enabled, "wip_title_tagging_enabled": row.wip_title_tagging_enabled,
"wip_soft_title_tagging_enabled": row.wip_soft_title_tagging_enabled,
}) })
@@ -179,6 +181,12 @@ async def update_import_settings():
return jsonify( return jsonify(
{"error": "wip_title_tagging_enabled must be a boolean"} {"error": "wip_title_tagging_enabled must be a boolean"}
), 400 ), 400
if "wip_soft_title_tagging_enabled" in body and not isinstance(
body["wip_soft_title_tagging_enabled"], bool
):
return jsonify(
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
), 400
async with get_session() as session: async with get_session() as session:
row = await ImportSettings.load(session) row = await ImportSettings.load(session)
+11 -1
View File
@@ -171,9 +171,19 @@ def make_celery() -> Celery:
}, },
"presentation-auto-apply-daily": { "presentation-auto-apply-daily": {
"task": "backend.app.tasks.ml.scheduled_presentation_auto_apply", "task": "backend.app.tasks.ml.scheduled_presentation_auto_apply",
"schedule": 86400.0, # auto-hide banner/editor chrome (#141); "schedule": 86400.0, # auto-hide banner chrome (#141);
# no-op unless presentation_auto_apply_enabled # no-op unless presentation_auto_apply_enabled
}, },
"process-auto-apply-daily": {
"task": "backend.app.tasks.ml.scheduled_process_auto_apply",
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
# no-op unless process_auto_apply_enabled (opt-in)
},
"soft-wip-conflict-audit-daily": {
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
# for review (#1474); no-op with no content heads
},
"prune-presentation-reviews-daily": { "prune-presentation-reviews-daily": {
"task": "backend.app.tasks.ml.prune_presentation_reviews", "task": "backend.app.tasks.ml.prune_presentation_reviews",
"schedule": 86400.0, # retention: drop resolved review flags >30d "schedule": 86400.0, # retention: drop resolved review flags >30d
+7
View File
@@ -126,6 +126,13 @@ class ImportSettings(Base):
wip_title_tagging_enabled: Mapped[bool] = mapped_column( wip_title_tagging_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=True, server_default="true", Boolean, nullable=False, default=True, server_default="true",
) )
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
# precision tier is opt-in (the ring-loud audit surfaces false positives).
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="false",
)
@classmethod @classmethod
async def load(cls, session) -> ImportSettings: async def load(cls, session) -> ImportSettings:
+28 -6
View File
@@ -85,12 +85,14 @@ class MLSettings(Base):
Float, nullable=False, default=0.95 Float, nullable=False, default=0.95
) )
# -- Presentation chrome auto-hide (#141) ------------------------------- # -- Presentation chrome auto-hide (#141) -------------------------------
# banner / editor screenshot auto-apply on the sweep with their OWN flat # `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
# threshold (decoupled from content-head graduation). Hiding is consequential # with its OWN flat threshold (decoupled from content-head graduation) and is
# so it runs HIGH. `wip` is never auto-applied. When an image would be # HIDDEN from the gallery. Hiding is consequential so it runs HIGH. When an
# auto-hidden but ALSO scores >= presentation_conflict_threshold on a content # image would be auto-hidden but ALSO scores >= presentation_conflict_threshold
# head, it's still hidden but flagged for review (PresentationReview) instead # on a content head, it's still hidden but flagged for review
# of buried silently. ON by default (opt-out); every auto-tag is reversible. # (PresentationReview, mode='chrome') instead of buried silently. ON by default
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
# screenshot` are no longer chrome — they went to the PROCESS path below.
presentation_auto_apply_enabled: Mapped[bool] = mapped_column( presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=True Boolean, nullable=False, default=True
) )
@@ -100,6 +102,26 @@ class MLSettings(Base):
presentation_conflict_threshold: Mapped[float] = mapped_column( presentation_conflict_threshold: Mapped[float] = mapped_column(
Float, nullable=False, default=0.50 Float, nullable=False, default=0.50
) )
# -- Process auto-apply (#1464) ----------------------------------------
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
# screenshots that must stay OUT of head/CCIP training but, unlike chrome,
# remain VISIBLE in the gallery (operator 2026-07-12). They auto-apply on the
# sweep with their OWN flat threshold and a PROVISIONAL source (`process_auto`,
# in training_data._AUTO_SOURCES) so the head NEVER trains on its own output —
# it learns only from title (`wip_title`) + manual labels, which breaks the
# runaway loop. When a process tag would be applied but the image ALSO scores
# >= process_conflict_threshold on a content head, it's flagged for review
# (PresentationReview, mode='process') rather than silently marked. OFF by
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
process_auto_apply_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False
)
process_auto_apply_threshold: Mapped[float] = mapped_column(
Float, nullable=False, default=0.90
)
process_conflict_threshold: Mapped[float] = mapped_column(
Float, nullable=False, default=0.50
)
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069); # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
# existing libraries keep their stored value until the operator re-embeds. # existing libraries keep their stored value until the operator re-embeds.
embedder_model_version: Mapped[str] = mapped_column( embedder_model_version: Mapped[str] = mapped_column(
+15 -7
View File
@@ -1,15 +1,17 @@
"""PresentationReview — an auto-hidden presentation tag that ALSO looked like """PresentationReview — a system-tag the auto-apply sweep applied that ALSO looked
real content, flagged for operator review (milestone 141). like real content, flagged for operator review (milestone 141 + #1464).
When the auto-apply sweep hides an image as chrome (banner / editor screenshot) When a sweep applies a system tag but the image ALSO scores highly on a content
but the image ALSO scores highly on a content head, it still hides it but records head, it still applies the tag but records this row so a review strip can surface
this row so the Hidden view can surface it ("⚠ also looks like <conflict tag>") it ("⚠ also looks like <conflict tag>"). Two modes (#1464): 'chrome' (banner —
for a keep-hidden / un-hide decision. Resolved rows are pruned by retention. image is HIDDEN, review is keep-hidden / un-hide) and 'process' (wip / editor
screenshot — image stays VISIBLE, review is confirm / remove-tag). Resolved rows
are pruned by retention.
""" """
from datetime import datetime from datetime import datetime
from sqlalchemy import DateTime, Float, ForeignKey, func from sqlalchemy import DateTime, Float, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from .base import Base from .base import Base
@@ -31,6 +33,12 @@ class PresentationReview(Base):
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True ForeignKey("tag.id", ondelete="SET NULL"), nullable=True
) )
conflict_score: Mapped[float] = mapped_column(Float, nullable=False) conflict_score: Mapped[float] = mapped_column(Float, nullable=False)
# Which sweep flagged this (#1464): 'chrome' (banner, hidden) or 'process'
# (wip / editor screenshot, shown). Drives which review strip surfaces it and
# what "resolve" means (un-hide vs remove-tag). Existing rows backfill 'chrome'.
mode: Mapped[str] = mapped_column(
String(16), nullable=False, default="chrome", server_default="chrome"
)
created_at: Mapped[datetime] = mapped_column( created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now() DateTime(timezone=True), nullable=False, server_default=func.now()
) )
+12 -7
View File
@@ -43,14 +43,19 @@ class TagKind(StrEnum):
# to keep historic tag rows queryable. # to keep historic tag rows queryable.
# The seeded system tags (migration 0075). PRESENTATION tags additionally # The seeded system tags (migration 0075). Two behavior groups (#1464):
# hide from whole-image similarity results — they cluster on UI chrome, not # CHROME (banner): clusters on UI chrome, not content → HIDDEN from the default
# content. `wip` is real art: only the training pipelines exclude it. # gallery + from similarity; auto-applied via the sweep's chrome mode.
# PROCESS (wip, editor screenshot): real-but-unfinished art / program screenshots
# → SHOWN in the gallery (operator 2026-07-12), but excluded from the Explore
# rabbit-hole; auto-applied via the sweep's process mode (provisional source,
# ring-loud review guard).
# ALL three are excluded from OTHER concepts' head/CCIP training (training-hygiene,
# keyed on is_system); a system tag's OWN head trains on them — that's what makes
# auto-flagging work.
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot") CHROME_SYSTEM_TAGS = ("banner",)
# `wip` marks real-but-unfinished art. It's kept in the gallery's own "similar" PROCESS_SYSTEM_TAGS = ("wip", "editor screenshot")
# results (#1274), but the Explore rabbit-hole opts to hide it (exclude_wip) so a
# browse doesn't keep surfacing work-in-progress (operator, 2026-07-08).
WIP_SYSTEM_TAG = "wip" WIP_SYSTEM_TAG = "wip"
image_tag = Table( image_tag = Table(
+51 -15
View File
@@ -31,7 +31,7 @@ from ..models import (
Tag, Tag,
TagPositiveConfirmation, TagPositiveConfirmation,
) )
from ..models.tag import PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG, image_tag from ..models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
from .pagination import decode_cursor, encode_cursor from .pagination import decode_cursor, encode_cursor
from .tag_query import ( from .tag_query import (
fandom_join_alias, fandom_join_alias,
@@ -396,6 +396,25 @@ def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
return [kept[i] for i in order] return [kept[i] for i in order]
def _reach_sample(rows, limit, reach):
"""From a distance-sorted candidate pool (nearest first), pick a spread of ranks
that MIXES near (tag the current cluster) and mid-far (escape it) BEFORE dedup +
MMR — the Explore "reach" dial (#1476).
reach in (0, 1]: the sampled span grows outward from the anchor (0.25→1.0 of the
pool), evenly strided from rank 0 so the nearest are still represented. In a
dense signature the nearest ranks are near-identical, so reaching farther is the
only way to hand MMR genuinely different content — MMR alone can't escape a pool
that's already all-near. reach<=0 or a small pool passes through unchanged."""
n = len(rows)
want = max(limit * 8, 100)
if reach <= 0 or n <= want:
return rows
span = int(min(1.0, 0.25 + 0.75 * reach) * n)
idx = sorted({min(int(i * span / want), n - 1) for i in range(want)})
return [rows[i] for i in idx]
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]: async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
"""Map image_id -> {"name","slug"} via the canonical """Map image_id -> {"name","slug"} via the canonical
image_record.artist_id (FC-2d-vii-c). Bounded by page size.""" image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
@@ -419,16 +438,17 @@ class GalleryService:
async def _hidden_tag_ids( async def _hidden_tag_ids(
self, include_hidden, tag_ids, tag_or_groups, self, include_hidden, tag_ids, tag_or_groups,
) -> list[int] | None: ) -> list[int] | None:
"""Presentation-chrome tag ids to implicitly exclude from a gallery query, """Chrome (banner) tag ids to implicitly exclude from a gallery query, or
or None. None when the caller asked to include hidden, when the operator None. None when the caller asked to include hidden, when the operator is
is explicitly filtering FOR a presentation tag (they clearly want to see explicitly filtering FOR a chrome tag (they clearly want to see it), or when
it), or when no presentation tags exist. (milestone 141)""" no chrome tags exist. (milestone 141; #1464: editor screenshot is now PROCESS
— shown — so only `banner` hides here.)"""
if include_hidden: if include_hidden:
return None return None
rows = await self.session.execute( rows = await self.session.execute(
select(Tag.id).where( select(Tag.id).where(
Tag.is_system.is_(True), Tag.is_system.is_(True),
Tag.name.in_(PRESENTATION_SYSTEM_TAGS), Tag.name.in_(CHROME_SYSTEM_TAGS),
) )
) )
pres = [r[0] for r in rows] pres = [r[0] for r in rows]
@@ -716,6 +736,7 @@ class GalleryService:
untagged: bool = False, no_artist: bool = False, untagged: bool = False, no_artist: bool = False,
date_from: datetime | None = None, date_to: datetime | None = None, date_from: datetime | None = None, date_to: datetime | None = None,
exclude_wip: bool = False, exclude_wip: bool = False,
reach: float = 0.0, exclude_ids: list[int] | None = None,
) -> list[GalleryImage] | None: ) -> list[GalleryImage] | None:
"""Visual "more like this": images near `image_id`'s SigLIP embedding """Visual "more like this": images near `image_id`'s SigLIP embedding
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result (pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
@@ -744,20 +765,27 @@ class GalleryService:
# wide pool there's nothing but the near-dupes to choose from. Widened # wide pool there's nothing but the near-dupes to choose from. Widened
# (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct # (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
# neighbourhoods to reach into for more variance (operator, 2026-07-01). # neighbourhoods to reach into for more variance (operator, 2026-07-01).
pool_n = min(400, max(limit * 8, 100)) # Explore's reach>0 (#1476) widens it a LOT more: in a dense signature the
# nearest few hundred are all near-identical, so far-enough candidates only
# exist deeper in the ranked pool. _reach_sample then strides across them.
if reach > 0:
pool_n = min(1000, max(limit * 25, 100))
else:
pool_n = min(400, max(limit * 8, 100))
distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding) distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
eff = _effective_date_col() eff = _effective_date_col()
stmt = select(ImageRecord, Post.post_date, eff.label("eff")) stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
stmt = _outer_join_primary_post(stmt) stmt = _outer_join_primary_post(stmt)
# Presentation images (banner / editor-screenshot system tags, #128) # Chrome (banner, #128) clusters on UI rather than content, so near any one
# cluster on UI chrome rather than content, so near any one of them # of them they'd fill the grid → excluded from CANDIDATES always (the anchor
# they'd fill the grid. Excluded from CANDIDATES only — the anchor # itself may be a banner). PROCESS art (wip / editor screenshot) stays
# itself may be a banner. `wip` stays surfaced here by default (real art; # surfaced here by default (real content; only the training pipelines exclude
# only the training pipelines exclude it), but the Explore rabbit-hole # it), but the Explore rabbit-hole passes exclude_wip to also drop the whole
# passes exclude_wip to also drop work-in-progress (operator, 2026-07-08). # process group so a browse doesn't keep surfacing work-in-progress
excluded_system_tags = PRESENTATION_SYSTEM_TAGS # (operator, 2026-07-08; #1464 — editor now rides with wip here).
excluded_system_tags = CHROME_SYSTEM_TAGS
if exclude_wip: if exclude_wip:
excluded_system_tags = (*PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG) excluded_system_tags = (*CHROME_SYSTEM_TAGS, *PROCESS_SYSTEM_TAGS)
presentation = ( presentation = (
select(image_tag.c.image_record_id) select(image_tag.c.image_record_id)
.join(Tag, Tag.id == image_tag.c.tag_id) .join(Tag, Tag.id == image_tag.c.tag_id)
@@ -771,6 +799,10 @@ class GalleryService:
ImageRecord.id != image_id, ImageRecord.id != image_id,
ImageRecord.id.not_in(presentation), ImageRecord.id.not_in(presentation),
) )
# Anti-revisit (#1476): the Explore walk passes its breadcrumb so already-
# walked images aren't re-served as neighbours — → can't loop you back in.
if exclude_ids:
stmt = stmt.where(ImageRecord.id.not_in(exclude_ids))
stmt = _apply_scope( stmt = _apply_scope(
stmt, tag_ids=tag_ids, post_id=None, stmt, tag_ids=tag_ids, post_id=None,
artist_id=artist_id, media_type=media_type, artist_id=artist_id, media_type=media_type,
@@ -780,6 +812,10 @@ class GalleryService:
) )
stmt = stmt.order_by(distance.asc()).limit(pool_n) stmt = stmt.order_by(distance.asc()).limit(pool_n)
rows = (await self.session.execute(stmt)).all() rows = (await self.session.execute(stmt)).all()
# Explore reach: stride across an outward-growing distance span so the pool
# handed to MMR spans near→mid-far, not just the tight cluster (#1476).
if reach > 0:
rows = _reach_sample(rows, limit, reach)
rows = _diversify_similar(src, rows, limit) rows = _diversify_similar(src, rows, limit)
artists = await _artists_for(self.session, [r[0].id for r in rows]) artists = await _artists_for(self.session, [r[0].id for r in rows])
return _gallery_images(rows, artists) return _gallery_images(rows, artists)
+22 -4
View File
@@ -47,7 +47,14 @@ from .attachment_store import AttachmentStore
from .audits import single_color from .audits import single_color
from .link_extract import extract_external_links from .link_extract import extract_external_links
from .thumbnailer import Thumbnailer from .thumbnailer import Thumbnailer
from .wip_title import apply_wip_image_tags, matches_wip_title, resolve_wip_tag_id from .wip_title import (
WIP_TITLE_SOFT_SOURCE,
WIP_TITLE_SOURCE,
apply_wip_image_tags,
matches_soft_wip_title,
matches_wip_title,
resolve_wip_tag_id,
)
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -999,7 +1006,9 @@ class Importer:
removal sticks. The existing catalogue is covered separately by the removal sticks. The existing catalogue is covered separately by the
operator-triggered backfill sweep. Gated by the settings toggle, and operator-triggered backfill sweep. Gated by the settings toggle, and
best-effort: any failure is logged, never allowed to fail the import.""" best-effort: any failure is logged, never allowed to fail the import."""
if not self.settings.wip_title_tagging_enabled: hard_on = self.settings.wip_title_tagging_enabled
soft_on = self.settings.wip_soft_title_tagging_enabled
if not (hard_on or soft_on):
return return
if record.primary_post_id is None: if record.primary_post_id is None:
return return
@@ -1007,13 +1016,22 @@ class Importer:
title = self.session.execute( title = self.session.execute(
select(Post.post_title).where(Post.id == record.primary_post_id) select(Post.post_title).where(Post.id == record.primary_post_id)
).scalar_one_or_none() ).scalar_one_or_none()
if not matches_wip_title(title): # HARD tier ("WIP"/"work in progress") wins — higher precision, and it
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
# that never trains (source wip_title_soft).
if hard_on and matches_wip_title(title):
source = WIP_TITLE_SOURCE
elif soft_on and matches_soft_wip_title(title):
source = WIP_TITLE_SOFT_SOURCE
else:
return return
if self._wip_tag_id is _UNSET: if self._wip_tag_id is _UNSET:
self._wip_tag_id = resolve_wip_tag_id(self.session) self._wip_tag_id = resolve_wip_tag_id(self.session)
if self._wip_tag_id is None: if self._wip_tag_id is None:
return return
apply_wip_image_tags(self.session, [record.id], self._wip_tag_id) apply_wip_image_tags(
self.session, [record.id], self._wip_tag_id, source=source
)
except Exception as exc: # noqa: BLE001 — a tag must never fail an import except Exception as exc: # noqa: BLE001 — a tag must never fail an import
log.warning( log.warning(
"wip-title auto-tag failed for image %s: %s", record.id, exc "wip-title auto-tag failed for image %s: %s", record.id, exc
+122 -21
View File
@@ -39,7 +39,7 @@ from ...models import (
TagPositiveConfirmation, TagPositiveConfirmation,
TagSuggestionRejection, TagSuggestionRejection,
) )
from ...models.tag import PRESENTATION_SYSTEM_TAGS, image_tag from ...models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
from .training_data import ( from .training_data import (
_AUTO_SOURCES, _AUTO_SOURCES,
_auto_apply_point, _auto_apply_point,
@@ -759,18 +759,42 @@ def auto_apply_sweep(
_PRESENTATION_SOURCE = "presentation_auto" _PRESENTATION_SOURCE = "presentation_auto"
_PROCESS_SOURCE = "process_auto"
# System-tag auto-apply modes (#1464). Both modes run the identical sweep — apply
# a system tag at a flat threshold with a PROVISIONAL source + a ring-loud review
# guard — and differ ONLY in which tags, which settings knobs, and which
# source/review-mode. 'chrome' (banner) is HIDDEN from the gallery; 'process'
# (wip / editor screenshot) stays VISIBLE (the hide is a gallery-query effect of
# the tag's group membership, not of this sweep).
_SWEEP_MODES = {
"chrome": {
"names": CHROME_SYSTEM_TAGS,
"enabled": "presentation_auto_apply_enabled",
"threshold": "presentation_auto_apply_threshold",
"conflict": "presentation_conflict_threshold",
"source": _PRESENTATION_SOURCE,
},
"process": {
"names": PROCESS_SYSTEM_TAGS,
"enabled": "process_auto_apply_enabled",
"threshold": "process_auto_apply_threshold",
"conflict": "process_conflict_threshold",
"source": _PROCESS_SOURCE,
},
}
def _presentation_heads(session: Session, embedding_version: str): def _system_tag_heads(session: Session, embedding_version: str, names):
"""Trained heads for the presentation chrome tags (banner / editor screenshot). """Trained heads for a system-tag group (chrome banner / process wip+editor).
They fire at the FLAT presentation threshold regardless of graduation — a head They fire at the group's FLAT threshold regardless of graduation — a head
exists once the operator has labelled enough chrome (head_min_positives).""" exists once the operator has labelled enough (head_min_positives)."""
return session.execute( return session.execute(
select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias) select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias)
.join(Tag, Tag.id == TagHead.tag_id) .join(Tag, Tag.id == TagHead.tag_id)
.where(TagHead.embedding_version == embedding_version) .where(TagHead.embedding_version == embedding_version)
.where(Tag.is_system.is_(True)) .where(Tag.is_system.is_(True))
.where(Tag.name.in_(PRESENTATION_SYSTEM_TAGS)) .where(Tag.name.in_(names))
).all() ).all()
@@ -802,27 +826,33 @@ def _valued_image_ids(session: Session) -> set[int]:
return {r[0] for r in rows} return {r[0] for r in rows}
def presentation_auto_apply_sweep(session: Session, dry_run: bool = False) -> dict: def system_tag_auto_apply_sweep(
"""Auto-hide presentation chrome (banner / editor screenshot) at the FLAT session: Session, *, mode: str, dry_run: bool = False
presentation threshold (#141) — NOT the per-head graduated threshold. Two ) -> dict:
guards keep it safe: (1) never hide an image carrying a human/confirmed content """Auto-apply a system-tag group at its FLAT threshold. mode='chrome' (banner,
tag; (2) if an image about to be hidden ALSO scores >= the conflict threshold #141) hides the image; mode='process' (wip / editor screenshot, #1464) keeps it
on a content head, still hide it but flag it (PresentationReview) so the Hidden VISIBLE — the ONLY difference is the tag group's gallery membership, not this
view surfaces "also looks like <X>" for review. No-op unless sweep. Two guards keep it safe: (1) never touch an image carrying a
presentation_auto_apply_enabled. numpy-only (no sklearn). Returns human/confirmed content tag; (2) if the image ALSO scores >= the conflict
{n_applied, n_flagged, concepts}.""" threshold on a content head, still apply but flag it (PresentationReview,
mode=<mode>) so the review strip surfaces "also looks like <X>". The source is
PROVISIONAL so the head never trains on its own output. No-op unless the mode's
enabled flag is set. numpy-only (no sklearn). Returns {n_applied, n_flagged,
concepts}."""
import numpy as np import numpy as np
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
cfg = _SWEEP_MODES[mode]
settings = _settings(session) settings = _settings(session)
if not dry_run and not settings.presentation_auto_apply_enabled: if not dry_run and not getattr(settings, cfg["enabled"]):
return {"n_applied": 0, "n_flagged": 0, "concepts": []} return {"n_applied": 0, "n_flagged": 0, "concepts": []}
ver = settings.embedder_model_version ver = settings.embedder_model_version
pres = _presentation_heads(session, ver) pres = _system_tag_heads(session, ver, cfg["names"])
if not pres: if not pres:
return {"n_applied": 0, "n_flagged": 0, "concepts": []} return {"n_applied": 0, "n_flagged": 0, "concepts": []}
thr = float(settings.presentation_auto_apply_threshold) thr = float(getattr(settings, cfg["threshold"]))
conflict_thr = float(settings.presentation_conflict_threshold) conflict_thr = float(getattr(settings, cfg["conflict"]))
source = cfg["source"]
Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres]) Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres])
bp = np.asarray([r.bias for r in pres], dtype=np.float32) bp = np.asarray([r.bias for r in pres], dtype=np.float32)
@@ -884,11 +914,13 @@ def presentation_auto_apply_sweep(session: Session, dry_run: bool = False) -> di
pg_insert(image_tag) pg_insert(image_tag)
.values( .values(
image_record_id=iid, tag_id=tid, image_record_id=iid, tag_id=tid,
source=_PRESENTATION_SOURCE, source=source,
) )
.on_conflict_do_nothing() .on_conflict_do_nothing()
) )
# Guard 2: also looks like content → hide but flag for review. # Guard 2: also looks like real content → still apply, but flag it
# for the review strip instead of silently marking (chrome hides,
# process stays visible — either way the operator gets a heads-up).
if Wc is not None and float(max_c[idx]) >= conflict_thr: if Wc is not None and float(max_c[idx]) >= conflict_thr:
n_flagged += 1 n_flagged += 1
if not dry_run: if not dry_run:
@@ -898,6 +930,7 @@ def presentation_auto_apply_sweep(session: Session, dry_run: bool = False) -> di
image_record_id=iid, tag_id=tid, image_record_id=iid, tag_id=tid,
conflict_tag_id=conf_tag_ids[int(arg_c[idx])], conflict_tag_id=conf_tag_ids[int(arg_c[idx])],
conflict_score=float(max_c[idx]), conflict_score=float(max_c[idx]),
mode=mode,
) )
.on_conflict_do_nothing() .on_conflict_do_nothing()
) )
@@ -914,6 +947,74 @@ def presentation_auto_apply_sweep(session: Session, dry_run: bool = False) -> di
} }
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
score >= the process conflict threshold on a content head are probably FINISHED
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
NOT remove the tag; the operator decides. No-op when there are no content heads.
numpy-only. Returns {n_scanned, n_flagged}."""
import numpy as np
from sqlalchemy.dialects.postgresql import insert as pg_insert
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
settings = _settings(session)
ver = settings.embedder_model_version
conflict_thr = float(settings.process_conflict_threshold)
conf = _conflict_heads(session, ver)
wip_id = resolve_wip_tag_id(session)
if not conf or wip_id is None:
return {"n_scanned": 0, "n_flagged": 0}
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
conf_tag_ids = [r.tag_id for r in conf]
soft_ids = [iid for (iid,) in session.execute(
select(image_tag.c.image_record_id)
.where(image_tag.c.tag_id == wip_id)
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
)]
# Skip images already flagged for this tag (idempotent re-runs).
flagged = {iid for (iid,) in session.execute(
select(PresentationReview.image_record_id)
.where(PresentationReview.tag_id == wip_id)
)}
soft_ids = [i for i in soft_ids if i not in flagged]
n_flagged = 0
scanned = 0
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
emb = _load_embeddings(session, chunk)
cids = [i for i in chunk if i in emb]
if not cids:
continue
scanned += len(cids)
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
cprobs = 1.0 / (1.0 + np.exp(-(Xn @ Wc.T + bc)))
max_c = cprobs.max(axis=1)
arg_c = cprobs.argmax(axis=1)
for k in range(len(cids)):
if float(max_c[k]) >= conflict_thr:
n_flagged += 1
if not dry_run:
session.execute(
pg_insert(PresentationReview)
.values(
image_record_id=cids[k], tag_id=wip_id,
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
conflict_score=float(max_c[k]),
mode="process",
)
.on_conflict_do_nothing()
)
if not dry_run:
session.commit()
return {"n_scanned": scanned, "n_flagged": n_flagged}
def retract_auto_applied_heads(session: Session) -> int: def retract_auto_applied_heads(session: Session) -> int:
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto' """Soft auto-apply (milestone 139): re-score every standing source='head_auto'
tag against its CURRENT head and REMOVE the ones now BELOW the head's tag against its CURRENT head and REMOVE the ones now BELOW the head's
+9 -1
View File
@@ -29,7 +29,15 @@ from ...models.tag import image_tag
# a CCIP reference) unless the operator confirms them (milestone 139). Keeping # a CCIP reference) unless the operator confirms them (milestone 139). Keeping
# auto-applied predictions out of training is what makes them "soft" — a misfire # auto-applied predictions out of training is what makes them "soft" — a misfire
# can't reinforce itself, so the retraction sweep can actually drop it. # can't reinforce itself, so the retraction sweep can actually drop it.
_AUTO_SOURCES = ("head_auto", "ccip_auto", "ml_auto", "presentation_auto") # `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
_AUTO_SOURCES = (
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
"wip_title_soft",
)
def _hygiene_excluded_ids(session: Session) -> set[int]: def _hygiene_excluded_ids(session: Session) -> set[int]:
+31 -7
View File
@@ -27,7 +27,13 @@ from ..models.tag import WIP_SYSTEM_TAG, Tag, image_tag
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other # image_tag.source stamped on title-heuristic WIP tags — distinct from the other
# apply sources so provenance stays legible and a future undo can target only these. # apply sources so provenance stays legible and a future undo can target only these.
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
WIP_TITLE_SOURCE = "wip_title" WIP_TITLE_SOURCE = "wip_title"
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
# surfaced by the ring-loud audit for review.
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress" # A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what # (space/underscore/hyphen separated). The letter-boundary lookarounds are what
@@ -39,11 +45,20 @@ _WIP_RE = re.compile(
re.IGNORECASE, re.IGNORECASE,
) )
# Coarse SQL prefilter for the backfill sweep — narrows the post scan to rows that # Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
# secondary because the soft source doesn't train the head and the ring-loud audit
# catches false positives.
_SOFT_WIP_RE = re.compile(
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
re.IGNORECASE,
)
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns. # COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
# MUST stay a SUPERSET of _WIP_RE (every regex match contains "wip" or # Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
# "work…progress") or the sweep would silently miss posts.
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%") WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter # Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under). # ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
@@ -51,12 +66,19 @@ _INSERT_CHUNK = 5000
def matches_wip_title(title: str | None) -> bool: def matches_wip_title(title: str | None) -> bool:
"""True when a post title explicitly marks it work-in-progress.""" """True when a post title explicitly marks it work-in-progress (HARD tier)."""
if not title: if not title:
return False return False
return _WIP_RE.search(title) is not None return _WIP_RE.search(title) is not None
def matches_soft_wip_title(title: str | None) -> bool:
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
if not title:
return False
return _SOFT_WIP_RE.search(title) is not None
def resolve_wip_tag_id(session: Session) -> int | None: def resolve_wip_tag_id(session: Session) -> int | None:
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent.""" """The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
return session.execute( return session.execute(
@@ -64,8 +86,10 @@ def resolve_wip_tag_id(session: Session) -> int | None:
).scalar_one_or_none() ).scalar_one_or_none()
def apply_wip_image_tags(session: Session, image_ids, tag_id: int) -> int: def apply_wip_image_tags(
"""Attach ``tag_id`` (source='wip_title') to each image id, idempotently — session: Session, image_ids, tag_id: int, *, source: str = WIP_TITLE_SOURCE
) -> int:
"""Attach ``tag_id`` (stamped with ``source``) to each image id, idempotently —
never disturbs an existing tag or its source. Returns the number of image_tag never disturbs an existing tag or its source. Returns the number of image_tag
rows newly inserted. Does NOT commit. rows newly inserted. Does NOT commit.
@@ -92,7 +116,7 @@ def apply_wip_image_tags(session: Session, image_ids, tag_id: int) -> int:
session.execute( session.execute(
pg_insert(image_tag) pg_insert(image_tag)
.values([ .values([
{"image_record_id": iid, "tag_id": tag_id, "source": WIP_TITLE_SOURCE} {"image_record_id": iid, "tag_id": tag_id, "source": source}
for iid in to_insert for iid in to_insert
]) ])
.on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"]) .on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"])
+61 -40
View File
@@ -1047,6 +1047,41 @@ def cleanup_old_download_events() -> int:
return result.rowcount or 0 return result.rowcount or 0
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
count newly applied."""
from ..models import Post
from ..models.image_provenance import ImageProvenance
from ..services.wip_title import apply_wip_image_tags
applied = 0
last_id = 0
while True:
rows = session.execute(
select(Post.id, Post.post_title)
.where(Post.id > last_id)
.where(Post.post_title.is_not(None))
.where(or_(*[Post.post_title.ilike(p) for p in prefilter]))
.order_by(Post.id.asc())
.limit(WIP_BACKFILL_PAGE)
).all()
if not rows:
break
last_id = rows[-1][0]
match_ids = [pid for pid, title in rows if matcher(title)]
if match_ids:
image_ids = session.execute(
select(ImageProvenance.image_record_id)
.where(ImageProvenance.post_id.in_(match_ids))
).scalars().all()
applied += apply_wip_image_tags(session, image_ids, tag_id, source=source)
session.commit()
return applied
@celery.task( @celery.task(
name="backend.app.tasks.maintenance.backfill_wip_title_tags", name="backend.app.tasks.maintenance.backfill_wip_title_tags",
# Coarse-prefiltered scan over posts; the candidate set is small on a typical # Coarse-prefiltered scan over posts; the candidate set is small on a typical
@@ -1054,32 +1089,32 @@ def cleanup_old_download_events() -> int:
soft_time_limit=1800, time_limit=2100, soft_time_limit=1800, time_limit=2100,
) )
def backfill_wip_title_tags() -> int: def backfill_wip_title_tags() -> int:
"""Scan EXISTING posts for explicit WIP titles and apply the `wip` system tag """Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
to their images — the operator-triggered back-catalogue catch-up for images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
title-based WIP tagging (task #1458). New imports are tagged live by the #1474 soft tier). New imports are tagged live by the importer; this covers the
importer; this covers everything already in the library. existing library.
Keyset-paginated over posts (restart-safe). A coarse SQL prefilter narrows to HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
titles that COULD match; the precise regex (matches_wip_title) confirms. scan); the SOFT tier (sketch/doodle, provisional source) runs only when
Idempotent-additive (ON CONFLICT DO NOTHING) — never disturbs an existing tag. wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
matching posts and silently undo a manual WIP removal, so it stays an explicit posts and silently undo a manual WIP removal, so it stays an explicit operator
operator action (Settings → "Scan existing posts for WIP titles"). Returns the action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
number of image-tag rows newly applied.
""" """
from ..models import Post from ..models import ImportSettings
from ..models.image_provenance import ImageProvenance
from ..services.wip_title import ( from ..services.wip_title import (
SOFT_WIP_TITLE_SQL_PREFILTER,
WIP_TITLE_SOFT_SOURCE,
WIP_TITLE_SOURCE,
WIP_TITLE_SQL_PREFILTER, WIP_TITLE_SQL_PREFILTER,
apply_wip_image_tags, matches_soft_wip_title,
matches_wip_title, matches_wip_title,
resolve_wip_tag_id, resolve_wip_tag_id,
) )
SessionLocal = _sync_session_factory() SessionLocal = _sync_session_factory()
applied = 0
last_id = 0
with SessionLocal() as session: with SessionLocal() as session:
tag_id = resolve_wip_tag_id(session) tag_id = resolve_wip_tag_id(session)
if tag_id is None: if tag_id is None:
@@ -1087,30 +1122,16 @@ def backfill_wip_title_tags() -> int:
"backfill_wip_title_tags: no `wip` system tag present; nothing to do" "backfill_wip_title_tags: no `wip` system tag present; nothing to do"
) )
return 0 return 0
like_a, like_b = WIP_TITLE_SQL_PREFILTER settings = ImportSettings.load_sync(session)
while True: applied = _backfill_wip_tier(
rows = session.execute( session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
select(Post.id, Post.post_title) WIP_TITLE_SOURCE,
.where(Post.id > last_id) )
.where(Post.post_title.is_not(None)) if settings.wip_soft_title_tagging_enabled:
.where(or_( applied += _backfill_wip_tier(
Post.post_title.ilike(like_a), session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
Post.post_title.ilike(like_b), WIP_TITLE_SOFT_SOURCE,
)) )
.order_by(Post.id.asc())
.limit(WIP_BACKFILL_PAGE)
).all()
if not rows:
break
last_id = rows[-1][0]
match_ids = [pid for pid, title in rows if matches_wip_title(title)]
if match_ids:
image_ids = session.execute(
select(ImageProvenance.image_record_id)
.where(ImageProvenance.post_id.in_(match_ids))
).scalars().all()
applied += apply_wip_image_tags(session, image_ids, tag_id)
session.commit()
if applied: if applied:
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied) log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
return applied return applied
+42 -6
View File
@@ -599,18 +599,54 @@ def scheduled_ccip_auto_apply() -> str:
soft_time_limit=1800, time_limit=2100, soft_time_limit=1800, time_limit=2100,
) )
def scheduled_presentation_auto_apply() -> str: def scheduled_presentation_auto_apply() -> str:
"""Auto-hide presentation chrome (banner / editor screenshot) on a daily """Auto-hide presentation chrome (banner) on a daily passive sweep (#141).
passive sweep (#141). No-op unless presentation_auto_apply_enabled. Idempotent No-op unless presentation_auto_apply_enabled. Idempotent — already-tagged images
— already-hidden images are skipped — so an interrupted run simply re-runs next are skipped — so an interrupted run simply re-runs next cycle (that IS the
cycle (that IS the recovery). Wall-clock bounded by the task time limits.""" recovery). Wall-clock bounded by the task time limits."""
from ..services.ml.heads import presentation_auto_apply_sweep from ..services.ml.heads import system_tag_auto_apply_sweep
SessionLocal = _sync_session_factory() SessionLocal = _sync_session_factory()
with SessionLocal() as session: with SessionLocal() as session:
result = presentation_auto_apply_sweep(session) result = system_tag_auto_apply_sweep(session, mode="chrome")
return f"applied={result['n_applied']} flagged={result['n_flagged']}" return f"applied={result['n_applied']} flagged={result['n_flagged']}"
@celery.task(
name="backend.app.tasks.ml.scheduled_process_auto_apply",
soft_time_limit=1800, time_limit=2100,
)
def scheduled_process_auto_apply() -> str:
"""Auto-apply the PROCESS system tags (wip / editor screenshot) on a daily
passive sweep (#1464) — provisional source, ring-loud review guard, image stays
VISIBLE. No-op unless process_auto_apply_enabled (opt-in). Idempotent —
already-tagged/rejected images are skipped — so an interrupted run just re-runs
next cycle (the recovery). Wall-clock bounded by the task time limits."""
from ..services.ml.heads import system_tag_auto_apply_sweep
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
result = system_tag_auto_apply_sweep(session, mode="process")
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
@celery.task(
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
soft_time_limit=1800, time_limit=2100,
)
def scheduled_soft_wip_conflict_audit() -> str:
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
auto-tags that ALSO look like real content for review. No-op when there are no
content heads; idempotent (already-flagged images skipped). Runs regardless of
the process-sweep toggle, since soft-title tags come from the importer, not that
sweep. Wall-clock bounded by the task time limits."""
from ..services.ml.heads import soft_wip_conflict_audit
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
result = soft_wip_conflict_audit(session)
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews") @celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
def prune_presentation_reviews() -> str: def prune_presentation_reviews() -> str:
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30 """Retention (rule 89): drop RESOLVED presentation-review flags older than 30
@@ -1,14 +1,15 @@
<template> <template>
<!-- Auto-hidden chrome that ALSO looked like real content surfaced PROACTIVELY <!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
atop the gallery whenever there's something to review (NOT gated on the like real content surfaced PROACTIVELY atop the gallery whenever there's
Show-hidden toggle, so misfires can't go unnoticed), most-concerning first, something to review (NOT gated on the Show-hidden toggle, so misfires can't
with keep / un-hide (#141). Renders nothing when there's nothing to review. --> go unnoticed), most-concerning first, with keep / remove (#141, #1464).
<section v-if="items.length" class="fc-review" aria-label="Hidden images to review"> Renders nothing when there's nothing to review. -->
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
<div class="fc-review__head"> <div class="fc-review__head">
<v-icon size="18" color="warning">mdi-alert-outline</v-icon> <v-icon size="18" color="warning">mdi-alert-outline</v-icon>
<span class="fc-review__title"> <span class="fc-review__title">
{{ items.length }} auto-hidden {{ items.length === 1 ? 'image' : 'images' }} {{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
may be real content — review before they stay hidden may be real content — review
</span> </span>
</div> </div>
<div class="fc-review__cards"> <div class="fc-review__cards">
@@ -26,16 +27,16 @@
> >
also looks like <strong>{{ it.conflict_name || 'content' }}</strong> also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
</div> </div>
<div class="fc-review-card__tag">hidden as {{ it.tag_name }}</div> <div class="fc-review-card__tag">{{ tagLine(it) }}</div>
<div class="fc-review-card__acts"> <div class="fc-review-card__acts">
<button <button
type="button" class="fc-review-btn fc-review-btn--keep" type="button" class="fc-review-btn fc-review-btn--keep"
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')" :disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
>Keep hidden</button> >{{ keepLabel(it) }}</button>
<button <button
type="button" class="fc-review-btn fc-review-btn--unhide" type="button" class="fc-review-btn fc-review-btn--unhide"
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')" :disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
>Un-hide</button> >{{ removeLabel(it) }}</button>
</div> </div>
</div> </div>
</div> </div>
@@ -54,6 +55,11 @@ const items = ref([])
const busy = ref([]) const busy = ref([])
function keyOf(it) { return `${it.image_id}:${it.tag_id}` } function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
async function load() { async function load() {
// Fetched unconditionally on mount — the strip prompts for pending misfires // Fetched unconditionally on mount — the strip prompts for pending misfires
@@ -71,7 +77,8 @@ async function resolve(it, action) {
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`) await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
items.value = items.value.filter((x) => keyOf(x) !== k) items.value = items.value.filter((x) => keyOf(x) !== k)
if (action === 'unhide') { if (action === 'unhide') {
toast({ text: `Un-hidden — “${it.tag_name}” removed; it'll train the head`, type: 'success' }) const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
} }
} catch (e) { } catch (e) {
toast({ toast({
+75 -6
View File
@@ -171,13 +171,13 @@
/> />
</div> </div>
<p class="fc-muted text-body-2 mb-3"> <p class="fc-muted text-body-2 mb-3">
Auto-hide banners and editor screenshots from the gallery once a head has Auto-hide <code>banner</code> chrome from the gallery once a head has
learned them ( {{ minPositives }} examples) and clears learned it ( {{ minPositives }} examples) and clears
{{ Math.round((presentationThresholdInput || 0) * 100) }}% confidence. {{ Math.round((presentationThresholdInput || 0) * 100) }}% confidence.
<code>wip</code> is never auto-hidden. If a hidden image also looks like (<code>wip</code> and <code>editor screenshot</code> are handled by the
real content ( {{ Math.round((presentationConflictInput || 0) * 100) }}% process auto-tagger below.) If a hidden image also looks like real content
on a content tag), it's flagged in the Hidden view instead of buried. ( {{ Math.round((presentationConflictInput || 0) * 100) }}% on a content
Every auto-hide is reversible. tag), it's flagged for review instead of buried. Every auto-hide is reversible.
</p> </p>
<div class="d-flex mb-3" style="gap: 12px;"> <div class="d-flex mb-3" style="gap: 12px;">
<v-text-field <v-text-field
@@ -195,6 +195,43 @@
</div> </div>
</div> </div>
<!-- Process auto-tagging (#1464): wip / editor screenshot -->
<div class="fc-auto mt-6">
<div class="d-flex align-center mb-1" style="gap: 10px;">
<v-icon size="18" color="accent">mdi-progress-wrench</v-icon>
<span class="fc-section-h">Auto-tag work-in-progress</span>
<v-switch
v-model="processEnabled" :loading="settingBusy" hide-details
density="compact" color="success" class="ml-auto"
@update:model-value="onToggleProcess"
/>
</div>
<p class="fc-muted text-body-2 mb-3">
Auto-tag <code>wip</code> and <code>editor screenshot</code> process art
once a head has learned them (≥ {{ minPositives }} examples) and clears
{{ Math.round((processThresholdInput || 0) * 100) }}% confidence. These stay
<strong>visible</strong> in the gallery — the tag just keeps them out of
training and the Explore rabbit-hole. Off by default. If a tagged image also
looks like real content (≥ {{ Math.round((processConflictInput || 0) * 100) }}%
on a content tag), it's flagged for review. Learns only from your titles +
manual tags, never its own guesses so it can't run away. Every tag reversible.
</p>
<div class="d-flex mb-3" style="gap: 12px;">
<v-text-field
v-model.number="processThresholdInput" label="Tag confidence"
type="number" min="0.5" max="0.999" step="0.01" density="compact"
hide-details style="max-width: 200px;" :disabled="settingBusy"
@change="onSaveProcess"
/>
<v-text-field
v-model.number="processConflictInput" label="Flag if content ≥"
type="number" min="0" max="1" step="0.05" density="compact"
hide-details style="max-width: 200px;" :disabled="settingBusy"
@change="onSaveProcess"
/>
</div>
</div>
<!-- Performance / tuning --> <!-- Performance / tuning -->
<div v-if="metricsConcepts.length" class="mt-5"> <div v-if="metricsConcepts.length" class="mt-5">
<div class="fc-section-h mb-1">How auto-apply is landing</div> <div class="fc-section-h mb-1">How auto-apply is landing</div>
@@ -258,6 +295,9 @@ let autoTimer = null
const presentationEnabled = ref(true) const presentationEnabled = ref(true)
const presentationThresholdInput = ref(0.90) const presentationThresholdInput = ref(0.90)
const presentationConflictInput = ref(0.50) const presentationConflictInput = ref(0.50)
const processEnabled = ref(false)
const processThresholdInput = ref(0.90)
const processConflictInput = ref(0.50)
const autoRunning = computed(() => autoStatus.value?.running_id != null) const autoRunning = computed(() => autoStatus.value?.running_id != null)
const lastSweep = computed(() => const lastSweep = computed(() =>
@@ -292,6 +332,9 @@ onMounted(async () => {
presentationEnabled.value = s.presentation_auto_apply_enabled ?? true presentationEnabled.value = s.presentation_auto_apply_enabled ?? true
presentationThresholdInput.value = s.presentation_auto_apply_threshold ?? 0.90 presentationThresholdInput.value = s.presentation_auto_apply_threshold ?? 0.90
presentationConflictInput.value = s.presentation_conflict_threshold ?? 0.50 presentationConflictInput.value = s.presentation_conflict_threshold ?? 0.50
processEnabled.value = s.process_auto_apply_enabled ?? false
processThresholdInput.value = s.process_auto_apply_threshold ?? 0.90
processConflictInput.value = s.process_conflict_threshold ?? 0.50
} catch { /* non-fatal */ } } catch { /* non-fatal */ }
await refresh() await refresh()
if (running.value) startPoll() if (running.value) startPoll()
@@ -402,6 +445,32 @@ async function onSavePresentation() {
settingBusy.value = false settingBusy.value = false
} }
} }
async function onToggleProcess(val) {
settingBusy.value = true
try {
await mlSettings.patchSettings({ process_auto_apply_enabled: !!val })
toast({ text: val ? 'WIP auto-tag on' : 'WIP auto-tag off', type: 'success' })
} catch (e) {
processEnabled.value = !val // revert the switch
toast({ text: `Could not update: ${e.message}`, type: 'error' })
} finally {
settingBusy.value = false
}
}
async function onSaveProcess() {
settingBusy.value = true
try {
await mlSettings.patchSettings({
process_auto_apply_threshold: Number(processThresholdInput.value),
process_conflict_threshold: Number(processConflictInput.value),
})
} catch (e) {
toast({ text: `Could not save: ${e.message}`, type: 'error' })
} finally {
settingBusy.value = false
}
}
function onPreview() { startSweep(true) } function onPreview() { startSweep(true) }
function onApplyNow() { startSweep(false) } function onApplyNow() { startSweep(false) }
async function startSweep(dryRun) { async function startSweep(dryRun) {
@@ -99,6 +99,17 @@
the Explore browse. Applies to new imports; run the scan below to catch the Explore browse. Applies to new imports; run the scan below to catch
posts already in your library. posts already in your library.
</div> </div>
<v-switch
v-model="local.wip_soft_title_tagging_enabled"
label="Also tag “sketch” / “doodle” titles (lower precision)"
density="compact" hide-details color="primary" @change="save"
/>
<div class="fc-help mb-3">
Extends the above to softer cues (<code>sketch</code>, <code>doodle</code>,
<code>scribble</code>). These stay <strong>visible</strong> and never train
the tagging model a daily audit flags any that actually look like finished
art for review. Off by default.
</div>
<v-btn <v-btn
variant="tonal" color="primary" size="small" variant="tonal" color="primary" size="small"
:loading="store.wipScanBusy" prepend-icon="mdi-magnify" :loading="store.wipScanBusy" prepend-icon="mdi-magnify"
@@ -139,6 +150,7 @@ const local = reactive({
skip_single_color: false, single_color_threshold: 0.95, skip_single_color: false, single_color_threshold: 0.95,
phash_threshold: 10, phash_threshold: 10,
wip_title_tagging_enabled: true, wip_title_tagging_enabled: true,
wip_soft_title_tagging_enabled: false,
}) })
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true }) watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
+21 -3
View File
@@ -25,6 +25,10 @@ export const useExploreStore = defineStore('explore', () => {
const cursor = ref(-1) const cursor = ref(-1)
const loading = ref(false) const loading = ref(false)
const error = ref(null) const error = ref(null)
// Reach (#1476): how far the walk reaches past the anchor's immediate cluster.
// 0 = nearest (can get stuck in a dense signature); ~0.4 default mixes in
// mid-far escape routes so the walk diversifies without hitting "Random image".
const reach = ref(0.4)
const inflight = useInflightToken() const inflight = useInflightToken()
@@ -46,7 +50,13 @@ export const useExploreStore = defineStore('explore', () => {
const body = await api.get('/api/gallery/similar', { const body = await api.get('/api/gallery/similar', {
// exclude_wip: keep work-in-progress out of the Explore rabbit-hole // exclude_wip: keep work-in-progress out of the Explore rabbit-hole
// (the gallery's own "similar" button still shows it) — operator 2026-07-08. // (the gallery's own "similar" button still shows it) — operator 2026-07-08.
params: { similar_to: numId, limit: NEIGHBOR_LIMIT, exclude_wip: 1 }, // reach + exclude_ids (#1476): reach past the dense cluster + never re-serve
// an already-walked image, so the walk keeps moving instead of getting stuck.
params: {
similar_to: numId, limit: NEIGHBOR_LIMIT, exclude_wip: 1,
reach: reach.value,
exclude_ids: breadcrumb.value.map((c) => c.id).join(','),
},
}) })
if (!t.isCurrent()) return if (!t.isCurrent()) return
neighbors.value = body.images || [] neighbors.value = body.images || []
@@ -113,6 +123,14 @@ export const useExploreStore = defineStore('explore', () => {
loading.value = false loading.value = false
} }
// Change how far the walk reaches and re-fetch the current anchor's neighbours
// with the new setting (the anchor + trail are unchanged — only the grid varies).
function setReach (v) {
reach.value = Math.max(0, Math.min(1, Number(v)))
const id = anchor.value?.id
if (id != null) anchorOn(id)
}
// --- TagPanel "host" surface --------------------------------------------- // --- TagPanel "host" surface ---------------------------------------------
// The anchor IS the current image (same /api/gallery/image/<id> payload the // The anchor IS the current image (same /api/gallery/image/<id> payload the
// modal uses), so these mirror the modal store's tag-CRUD, targeting the // modal uses), so these mirror the modal store's tag-CRUD, targeting the
@@ -189,8 +207,8 @@ export const useExploreStore = defineStore('explore', () => {
function close () {} function close () {}
return { return {
anchor, neighbors, breadcrumb, cursor, loading, error, NEIGHBOR_LIMIT, anchor, neighbors, breadcrumb, cursor, loading, error, NEIGHBOR_LIMIT, reach,
anchorOn, reset, backTarget, forwardTarget, anchorOn, reset, backTarget, forwardTarget, setReach,
// host surface // host surface
current, currentImageId, current, currentImageId,
reloadTags, addExistingTag, removeTag, createAndAdd, close, reloadTags, addExistingTag, removeTag, createAndAdd, close,
+28
View File
@@ -31,6 +31,20 @@
<img :src="c.thumbnail_url" alt="" loading="lazy" /> <img :src="c.thumbnail_url" alt="" loading="lazy" />
</button> </button>
<div class="fc-ex__trail-actions"> <div class="fc-ex__trail-actions">
<!-- Reach (#1476): how far each step reaches past the anchor's cluster.
Raise it to break out of a dense signature without hitting Random. -->
<div
class="fc-ex__reach"
title="How far each step reaches — raise it to escape a dense cluster without going fully random"
>
<v-icon size="16" color="accent">mdi-map-marker-distance</v-icon>
<v-slider
:model-value="store.reach" @end="store.setReach"
:min="0" :max="1" :step="0.2" hide-details density="compact"
color="accent" class="fc-ex__reach-slider"
/>
<span class="fc-muted fc-ex__reach-label">{{ reachLabel }}</span>
</div>
<!-- Active retrain right where you tag: fold the +/- you just gave <!-- Active retrain right where you tag: fold the +/- you just gave
into the heads without a trip to Settings (the nightly beat is the into the heads without a trip to Settings (the nightly beat is the
passive cadence). --> passive cadence). -->
@@ -153,6 +167,12 @@ const modal = useModalStore()
const anchorId = computed(() => route.params.imageId || null) const anchorId = computed(() => route.params.imageId || null)
const isVideo = computed(() => !!store.anchor?.mime?.startsWith('video/')) const isVideo = computed(() => !!store.anchor?.mime?.startsWith('video/'))
const reachLabel = computed(() => {
const r = store.reach
if (r <= 0.15) return 'Near'
if (r <= 0.55) return 'Varied'
return 'Far'
})
// #1206: hovering a suggestion in the rail highlights the crop it came from on // #1206: hovering a suggestion in the rail highlights the crop it came from on
// the anchor image (same provide/inject as the modal viewer). // the anchor image (same provide/inject as the modal viewer).
@@ -296,6 +316,14 @@ onUnmounted(() => {
.fc-ex__trail-actions { .fc-ex__trail-actions {
margin-left: auto; display: flex; align-items: center; gap: 4px; flex: 0 0 auto; margin-left: auto; display: flex; align-items: center; gap: 4px; flex: 0 0 auto;
} }
.fc-ex__reach {
display: flex; align-items: center; gap: 6px;
margin-right: 8px;
}
.fc-ex__reach-slider { width: 96px; }
.fc-ex__reach-label {
font-size: 12px; min-width: 44px; text-align: left;
}
/* The three panes fill the remaining height; each scrolls on its own. /* The three panes fill the remaining height; each scrolls on its own.
grid-template-rows: minmax(0, 1fr) BOUNDS the single row to the container grid-template-rows: minmax(0, 1fr) BOUNDS the single row to the container
+9 -5
View File
@@ -82,24 +82,28 @@ async def _system_tag(db, name):
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_scroll_hides_presentation_chrome_by_default(db): async def test_scroll_hides_presentation_chrome_by_default(db):
# banner / editor screenshot (presentation system tags) are hidden from the # banner (chrome) is hidden from the default gallery; wip AND editor screenshot
# default gallery; wip (also a system tag) is NOT — it's real, in-progress # (the PROCESS system tags) are NOT — they're real art / process shots that stay
# art (milestone 141). Seeded system tags survive the harness TRUNCATE. # visible (milestone 141 + #1464). Seeded system tags survive the harness TRUNCATE.
imgs = await _seed_images(db, 3, sha_prefix="p") imgs = await _seed_images(db, 4, sha_prefix="p")
banner = await _system_tag(db, "banner") banner = await _system_tag(db, "banner")
wip = await _system_tag(db, "wip") wip = await _system_tag(db, "wip")
editor = await _system_tag(db, "editor screenshot")
await db.execute(image_tag.insert().values( await db.execute(image_tag.insert().values(
image_record_id=imgs[0].id, tag_id=banner.id, source="manual")) image_record_id=imgs[0].id, tag_id=banner.id, source="manual"))
await db.execute(image_tag.insert().values( await db.execute(image_tag.insert().values(
image_record_id=imgs[1].id, tag_id=wip.id, source="manual")) image_record_id=imgs[1].id, tag_id=wip.id, source="manual"))
await db.execute(image_tag.insert().values(
image_record_id=imgs[3].id, tag_id=editor.id, source="manual"))
await db.flush() await db.flush()
svc = GalleryService(db) svc = GalleryService(db)
# Default: the banner image is hidden; the wip image + the plain image stay. # Default: only the banner image is hidden; wip + editor + plain all stay.
default_ids = {i.id for i in (await svc.scroll(cursor=None, limit=10)).images} default_ids = {i.id for i in (await svc.scroll(cursor=None, limit=10)).images}
assert imgs[0].id not in default_ids # banner hidden assert imgs[0].id not in default_ids # banner hidden
assert imgs[1].id in default_ids # wip visible assert imgs[1].id in default_ids # wip visible
assert imgs[2].id in default_ids # plain visible assert imgs[2].id in default_ids # plain visible
assert imgs[3].id in default_ids # editor screenshot visible (#1464)
# include_hidden surfaces the banner image (the Hidden view). # include_hidden surfaces the banner image (the Hidden view).
shown = {i.id for i in ( shown = {i.id for i in (
+17
View File
@@ -170,6 +170,23 @@ async def test_similar_respects_limit(db):
assert len(res) == 2 assert len(res) == 2
@pytest.mark.asyncio
async def test_similar_exclude_ids_drops_walked(db):
"""Explore passes its breadcrumb as exclude_ids so already-walked images
aren't re-served as neighbours (#1476). reach>0 runs cleanly too (small pool
→ the sampler passes through)."""
src = await _img(db, 1, _vec(1, 0))
walked = await _img(db, 2, _vec(1, 0.05))
fresh = await _img(db, 3, _vec(1, 0.3))
svc = GalleryService(db)
res = await svc.similar(src.id, limit=10, exclude_ids=[walked.id])
ids = {i.id for i in res}
assert walked.id not in ids
assert fresh.id in ids
res_reach = await svc.similar(src.id, limit=10, reach=1.0)
assert fresh.id in {i.id for i in res_reach}
# --- API --- # --- API ---
@pytest.mark.asyncio @pytest.mark.asyncio
+6 -6
View File
@@ -15,7 +15,7 @@ from backend.app.models import (
from backend.app.models.tag import image_tag from backend.app.models.tag import image_tag
from backend.app.services.ml.heads import ( from backend.app.services.ml.heads import (
auto_apply_sweep, auto_apply_sweep,
presentation_auto_apply_sweep, system_tag_auto_apply_sweep,
) )
pytestmark = pytest.mark.integration pytestmark = pytest.mark.integration
@@ -70,7 +70,7 @@ def test_presentation_sweep_hides_chrome(db_sync):
_head(db_sync, banner.id, 0, weight=3.0) _head(db_sync, banner.id, 0, weight=3.0)
img = _img(db_sync, "a" * 64, _emb(0)) img = _img(db_sync, "a" * 64, _emb(0))
db_sync.commit() db_sync.commit()
res = presentation_auto_apply_sweep(db_sync) res = system_tag_auto_apply_sweep(db_sync, mode="chrome")
assert res["n_applied"] == 1 assert res["n_applied"] == 1
assert _source(db_sync, img.id, banner.id) == "presentation_auto" assert _source(db_sync, img.id, banner.id) == "presentation_auto"
@@ -86,7 +86,7 @@ def test_presentation_sweep_hard_skips_valued_image(db_sync):
db_sync.execute(image_tag.insert().values( db_sync.execute(image_tag.insert().values(
image_record_id=img.id, tag_id=content.id, source="manual")) image_record_id=img.id, tag_id=content.id, source="manual"))
db_sync.commit() db_sync.commit()
res = presentation_auto_apply_sweep(db_sync) res = system_tag_auto_apply_sweep(db_sync, mode="chrome")
assert res["n_applied"] == 0 assert res["n_applied"] == 0
assert _source(db_sync, img.id, banner.id) is None assert _source(db_sync, img.id, banner.id) is None
@@ -102,7 +102,7 @@ def test_presentation_sweep_flags_conflict(db_sync):
_head(db_sync, content.id, 0, weight=1.0) # content head also fires _head(db_sync, content.id, 0, weight=1.0) # content head also fires
img = _img(db_sync, "c" * 64, _emb(0)) img = _img(db_sync, "c" * 64, _emb(0))
db_sync.commit() db_sync.commit()
res = presentation_auto_apply_sweep(db_sync) res = system_tag_auto_apply_sweep(db_sync, mode="chrome")
assert res["n_applied"] == 1 assert res["n_applied"] == 1
assert res["n_flagged"] == 1 assert res["n_flagged"] == 1
assert _source(db_sync, img.id, banner.id) == "presentation_auto" assert _source(db_sync, img.id, banner.id) == "presentation_auto"
@@ -123,7 +123,7 @@ def test_presentation_sweep_disabled_is_noop(db_sync):
_head(db_sync, banner.id, 0, weight=3.0) _head(db_sync, banner.id, 0, weight=3.0)
img = _img(db_sync, "d" * 64, _emb(0)) img = _img(db_sync, "d" * 64, _emb(0))
db_sync.commit() db_sync.commit()
res = presentation_auto_apply_sweep(db_sync) res = system_tag_auto_apply_sweep(db_sync, mode="chrome")
assert res["n_applied"] == 0 assert res["n_applied"] == 0
assert _source(db_sync, img.id, banner.id) is None assert _source(db_sync, img.id, banner.id) is None
@@ -134,7 +134,7 @@ def test_presentation_sweep_ignores_wip(db_sync):
_head(db_sync, wip.id, 0, weight=3.0) _head(db_sync, wip.id, 0, weight=3.0)
img = _img(db_sync, "e" * 64, _emb(0)) img = _img(db_sync, "e" * 64, _emb(0))
db_sync.commit() db_sync.commit()
res = presentation_auto_apply_sweep(db_sync) res = system_tag_auto_apply_sweep(db_sync, mode="chrome")
assert res["n_applied"] == 0 assert res["n_applied"] == 0
assert _source(db_sync, img.id, wip.id) is None assert _source(db_sync, img.id, wip.id) is None
+188
View File
@@ -0,0 +1,188 @@
"""Process-group auto-apply sweep (#1464): wip / editor screenshot auto-tag at a
flat threshold with a PROVISIONAL source (`process_auto`) so the head never trains
on its own output, and stay VISIBLE (unlike chrome). Mirrors the chrome guards.
numpy-only (no sklearn), tested directly via the sync session."""
import pytest
from sqlalchemy import select
from backend.app.models import (
ImageRecord,
MLSettings,
PresentationReview,
Tag,
TagHead,
TagKind,
)
from backend.app.models.tag import image_tag
from backend.app.services.ml.heads import system_tag_auto_apply_sweep
from backend.app.services.ml.training_data import _ids_with_tag
pytestmark = pytest.mark.integration
def _emb(slot: int) -> list[float]:
v = [0.0] * 1152
v[slot] = 3.0
return v
def _img(db, sha: str, emb) -> ImageRecord:
img = ImageRecord(
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
width=1, height=1, origin="imported_filesystem",
integrity_status="unknown", siglip_embedding=emb,
)
db.add(img)
db.flush()
return img
def _head(db, tag_id: int, slot: int, *, weight=1.0):
s = db.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one()
w = [0.0] * 1152
w[slot] = weight
db.add(TagHead(
tag_id=tag_id, embedding_version=s.embedder_model_version,
weights=w, bias=0.0, suggest_threshold=0.5, auto_apply_threshold=0.5,
n_pos=60, n_neg=90, ap=0.9, precision_cv=0.98, recall=0.7,
))
def _system_tag(db, name):
return db.execute(
select(Tag).where(Tag.is_system.is_(True), Tag.name == name)
).scalar_one()
def _enable_process(db):
# process auto-apply is opt-in (default False) — turn it on for these tests.
db.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one().process_auto_apply_enabled = True
def _source(db, image_id, tag_id):
return db.execute(
select(image_tag.c.source)
.where(image_tag.c.image_record_id == image_id)
.where(image_tag.c.tag_id == tag_id)
).scalar_one_or_none()
def test_process_sweep_applies_wip_and_editor(db_sync):
_enable_process(db_sync)
wip = _system_tag(db_sync, "wip")
editor = _system_tag(db_sync, "editor screenshot")
_head(db_sync, wip.id, 0, weight=3.0)
_head(db_sync, editor.id, 1, weight=3.0)
w_img = _img(db_sync, "a" * 64, _emb(0))
e_img = _img(db_sync, "b" * 64, _emb(1))
db_sync.commit()
res = system_tag_auto_apply_sweep(db_sync, mode="process")
assert res["n_applied"] == 2
assert _source(db_sync, w_img.id, wip.id) == "process_auto"
assert _source(db_sync, e_img.id, editor.id) == "process_auto"
def test_process_sweep_disabled_by_default_is_noop(db_sync):
# process_auto_apply_enabled defaults False (opt-in) — no enable = no-op.
wip = _system_tag(db_sync, "wip")
_head(db_sync, wip.id, 0, weight=3.0)
img = _img(db_sync, "c" * 64, _emb(0))
db_sync.commit()
res = system_tag_auto_apply_sweep(db_sync, mode="process")
assert res["n_applied"] == 0
assert _source(db_sync, img.id, wip.id) is None
def test_process_sweep_skips_valued_image(db_sync):
# Guard 1: never auto-apply to an image the operator already content-tagged.
_enable_process(db_sync)
wip = _system_tag(db_sync, "wip")
_head(db_sync, wip.id, 0, weight=3.0)
content = Tag(name="mychar", kind=TagKind.character)
db_sync.add(content)
db_sync.flush()
img = _img(db_sync, "d" * 64, _emb(0))
db_sync.execute(image_tag.insert().values(
image_record_id=img.id, tag_id=content.id, source="manual"))
db_sync.commit()
res = system_tag_auto_apply_sweep(db_sync, mode="process")
assert res["n_applied"] == 0
assert _source(db_sync, img.id, wip.id) is None
def test_process_sweep_flags_conflict_with_process_mode(db_sync):
# Guard 2: also scores high on a content head → still applied, but flagged
# for review with mode='process' (the ring-loud guard).
_enable_process(db_sync)
wip = _system_tag(db_sync, "wip")
_head(db_sync, wip.id, 0, weight=3.0)
content = Tag(name="looksreal", kind=TagKind.general)
db_sync.add(content)
db_sync.flush()
_head(db_sync, content.id, 0, weight=1.0)
img = _img(db_sync, "e" * 64, _emb(0))
db_sync.commit()
res = system_tag_auto_apply_sweep(db_sync, mode="process")
assert res["n_applied"] == 1
assert res["n_flagged"] == 1
flag = db_sync.execute(
select(PresentationReview).where(
PresentationReview.image_record_id == img.id,
PresentationReview.tag_id == wip.id,
)
).scalar_one()
assert flag.mode == "process"
assert flag.conflict_tag_id == content.id
def test_process_auto_source_never_trains_head(db_sync):
# The runaway break: provisional wip tags (process sweep 'process_auto', soft
# title 'wip_title_soft') are NOT training positives; a HARD title-heuristic /
# manual one IS. So the head learns only from trusted labels, never its own
# output or the low-precision sketch/doodle tier (#1464 + #1474).
wip = _system_tag(db_sync, "wip")
auto_img = _img(db_sync, "f" * 64, _emb(0))
soft_img = _img(db_sync, "9" * 64, _emb(2))
title_img = _img(db_sync, "0" * 64, _emb(1))
db_sync.execute(image_tag.insert().values(
image_record_id=auto_img.id, tag_id=wip.id, source="process_auto"))
db_sync.execute(image_tag.insert().values(
image_record_id=soft_img.id, tag_id=wip.id, source="wip_title_soft"))
db_sync.execute(image_tag.insert().values(
image_record_id=title_img.id, tag_id=wip.id, source="wip_title"))
db_sync.commit()
positives = set(_ids_with_tag(db_sync, wip.id))
assert title_img.id in positives # trusted HARD label trains the head
assert auto_img.id not in positives # its own auto-applied output does NOT
assert soft_img.id not in positives # low-precision soft tier does NOT
def test_soft_wip_conflict_audit_flags_ring_loud(db_sync):
# A soft-tagged image (sketch/doodle title) that ALSO scores high on a content
# head is probably finished art mis-tagged — flagged for review; a quiet one is not.
from backend.app.services.ml.heads import soft_wip_conflict_audit
s = db_sync.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one()
s.process_conflict_threshold = 0.6
wip = _system_tag(db_sync, "wip")
content = Tag(name="looksreal", kind=TagKind.general)
db_sync.add(content)
db_sync.flush()
_head(db_sync, content.id, 0, weight=1.0) # sigmoid(1)=0.73 > 0.6 conflict
ring = _img(db_sync, "1" * 64, _emb(0)) # scores on the content head
quiet = _img(db_sync, "2" * 64, _emb(5)) # orthogonal → 0.5 < 0.6
for img in (ring, quiet):
db_sync.execute(image_tag.insert().values(
image_record_id=img.id, tag_id=wip.id, source="wip_title_soft"))
db_sync.commit()
res = soft_wip_conflict_audit(db_sync)
assert res["n_flagged"] == 1
flag = db_sync.execute(
select(PresentationReview).where(PresentationReview.image_record_id == ring.id)
).scalar_one()
assert flag.mode == "process"
assert flag.conflict_tag_id == content.id
assert db_sync.execute(
select(PresentationReview).where(PresentationReview.image_record_id == quiet.id)
).scalar_one_or_none() is None
+32
View File
@@ -0,0 +1,32 @@
"""Explore reach sampler (#1476) — pure unit tests for `_reach_sample`, which picks
a distance-rank spread so the neighbour pool handed to MMR spans near→mid-far and
the walk can escape a dense cluster. No DB. `_reach_sample` only indexes rows, so
plain ints stand in for the (ImageRecord, ...) tuples."""
from backend.app.services.gallery_service import _reach_sample
def test_reach_zero_or_negative_passes_through():
rows = list(range(1000))
assert _reach_sample(rows, 40, 0.0) is rows
assert _reach_sample(rows, 40, -1.0) is rows
def test_small_pool_passes_through():
# n <= want (limit*8 = 320) → nothing to reach into.
rows = list(range(50))
assert _reach_sample(rows, 40, 1.0) is rows
def test_higher_reach_reaches_deeper_ranks():
rows = list(range(1000))
near = _reach_sample(rows, 40, 0.2)
far = _reach_sample(rows, 40, 1.0)
# Both keep the nearest rank (stride starts at 0) so you can still tag the cluster.
assert near[0] == 0
assert far[0] == 0
# But higher reach samples genuinely farther ranks.
assert max(far) > max(near)
assert max(far) >= 900 # reach=1 spans (almost) the whole pool
assert max(near) <= 550 # reach=0.2 stays in the near half
# Never runs past the pool.
assert max(far) <= len(rows) - 1
+28 -1
View File
@@ -8,7 +8,7 @@ negative cases (substrings like ``swipe`` / ``wiped``) are the load-bearing ones
""" """
import pytest import pytest
from backend.app.services.wip_title import matches_wip_title from backend.app.services.wip_title import matches_soft_wip_title, matches_wip_title
@pytest.mark.parametrize("title", [ @pytest.mark.parametrize("title", [
@@ -44,6 +44,33 @@ def test_matches_positive(title):
"finished at last", "finished at last",
"Kawips diner", # 'wip' mid-word "Kawips diner", # 'wip' mid-word
"swipright", "swipright",
"quick sketch of Nami", # soft cue — NOT a HARD WIP match
]) ])
def test_matches_negative(title): def test_matches_negative(title):
assert matches_wip_title(title) is False assert matches_wip_title(title) is False
@pytest.mark.parametrize("title", [
"quick sketch",
"Nami sketch",
"morning doodle",
"some doodles",
"sketches from today",
"a little scribble",
"SKETCH",
])
def test_soft_matches_positive(title):
assert matches_soft_wip_title(title) is True
@pytest.mark.parametrize("title", [
None,
"",
"sketchbook tour", # 'sketch' inside sketchbook — must NOT match
"kadoodle mascot", # 'doodle' mid-word
"the final piece",
"WIP", # a HARD cue is not a SOFT cue
"prescribed colours", # 'scrib' inside prescribed — must NOT match
])
def test_soft_matches_negative(title):
assert matches_soft_wip_title(title) is False
+16 -1
View File
@@ -9,7 +9,11 @@ from sqlalchemy import select
from backend.app.celery_app import celery from backend.app.celery_app import celery
from backend.app.models import Artist, ImageProvenance, ImageRecord, Post, Source from backend.app.models import Artist, ImageProvenance, ImageRecord, Post, Source
from backend.app.models.tag import image_tag from backend.app.models.tag import image_tag
from backend.app.services.wip_title import apply_wip_image_tags, resolve_wip_tag_id from backend.app.services.wip_title import (
WIP_TITLE_SOFT_SOURCE,
apply_wip_image_tags,
resolve_wip_tag_id,
)
pytestmark = pytest.mark.integration pytestmark = pytest.mark.integration
@@ -104,3 +108,14 @@ def test_backfill_tags_only_wip_titled_posts(db_sync):
# Idempotent: a second sweep finds the tag already present and applies nothing. # Idempotent: a second sweep finds the tag already present and applies nothing.
assert backfill_wip_title_tags.apply().get() == 0 assert backfill_wip_title_tags.apply().get() == 0
def test_apply_soft_source_stamps_wip_title_soft(db_sync):
# The soft tier (#1474) stamps a distinct provisional source.
tag_id = resolve_wip_tag_id(db_sync)
rec = _img(db_sync)
db_sync.commit()
assert apply_wip_image_tags(
db_sync, [rec.id], tag_id, source=WIP_TITLE_SOFT_SOURCE
) == 1
assert _wip_source(db_sync, rec.id, tag_id) == "wip_title_soft"