SubscribeStar .art age fix, modal tag-flow sync, system tags / training hygiene (#128) #190

Merged
bvandeusen merged 9 commits from dev into main 2026-07-03 09:16:16 -04:00
29 changed files with 869 additions and 63 deletions
+60
View File
@@ -0,0 +1,60 @@
"""tag.is_system + seed the three hygiene system tags
Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a
character poison that character's head and CCIP references; banners/editor
screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the
product ships — not operator configuration — so the seed lives here.
Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive,
matching TagService.rename's collision stance) instead of inserting a
duplicate, so an operator who already tagged `wip` keeps their applications.
Revision ID: 0075
Revises: 0074
Create Date: 2026-07-03
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0075"
down_revision: Union[str, None] = "0074"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
def upgrade() -> None:
op.add_column(
"tag",
sa.Column(
"is_system", sa.Boolean(), nullable=False,
server_default=sa.false(),
),
)
conn = op.get_bind()
for name in SYSTEM_TAG_NAMES:
adopted = conn.execute(
sa.text(
"UPDATE tag SET is_system = true "
"WHERE lower(name) = lower(:name) AND kind = 'general'"
),
{"name": name},
)
if adopted.rowcount == 0:
conn.execute(
sa.text(
"INSERT INTO tag (name, kind, is_system) "
"VALUES (:name, 'general', true)"
),
{"name": name},
)
def downgrade() -> None:
# The seeded rows survive as ordinary general tags — dropping the flag is
# enough to disarm the mechanism, and deleting rows would orphan any
# operator applications made while the flag existed.
op.drop_column("tag", "is_system")
+4
View File
@@ -156,6 +156,10 @@ async def tag_delete(tag_id: int):
)
except LookupError:
return _bad("not_found", status=404)
except ValueError as exc:
# System tags (#128) — the training-hygiene machinery keys on
# these rows.
return _bad("system_tag", detail=str(exc))
return jsonify(result)
+2
View File
@@ -324,6 +324,7 @@ async def get_tag(tag_id: int):
"name": tag.name,
"kind": tag.kind.value,
"fandom_id": tag.fandom_id,
"is_system": tag.is_system,
}
)
@@ -390,6 +391,7 @@ async def update_tag(tag_id: int):
"name": tag.name,
"kind": tag.kind.value,
"fandom_id": tag.fandom_id,
"is_system": tag.is_system,
}
)
+16
View File
@@ -10,6 +10,7 @@ from datetime import datetime
from enum import StrEnum
from sqlalchemy import (
Boolean,
CheckConstraint,
Column,
DateTime,
@@ -17,6 +18,7 @@ from sqlalchemy import (
Integer,
String,
Table,
false,
func,
)
from sqlalchemy import (
@@ -41,6 +43,12 @@ class TagKind(StrEnum):
# to keep historic tag rows queryable.
# The seeded system tags (migration 0075). PRESENTATION tags additionally
# hide from whole-image similarity results — they cluster on UI chrome, not
# content. `wip` is real art: only the training pipelines exclude it.
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot")
image_tag = Table(
"image_tag",
Base.metadata,
@@ -74,6 +82,14 @@ class Tag(Base):
fandom_id: Mapped[int | None] = mapped_column(
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
)
# System tags ship with FC (wip / banner / editor screenshot, seeded in
# migration 0075) and drive the training-hygiene exclusions: images
# carrying one are excluded from OTHER concepts' head training and from
# CCIP identity references. The mechanism keys on these exact rows, so
# they're protected from rename/merge-away/re-fandom in TagService.
is_system: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=false()
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
+13 -3
View File
@@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.orm import Session, aliased
from ..models import (
@@ -203,6 +203,9 @@ def _unused_tag_conditions() -> list:
Tag.id.not_in(used_via_series),
Tag.id.not_in(used_via_chapter),
Tag.id.not_in(used_via_fandom),
# System tags (#128) ship with zero applications and must survive a
# prune — the training-hygiene machinery keys on the rows.
Tag.is_system.is_(False),
]
@@ -402,6 +405,8 @@ def delete_tag(session: Session, *, tag_id: int) -> dict:
tag = session.get(Tag, tag_id)
if tag is None:
raise LookupError(f"tag id not found: {tag_id}")
if tag.is_system:
raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted")
associations_count = count_tag_associations(session, tag_id=tag_id)
info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
session.delete(tag)
@@ -731,7 +736,10 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
heads retrain from whatever the operator re-tags. (The API route gates the
live run behind a preview-derived confirm token for exactly this reason.)
PRESERVED: fandom + series tags and their series_page ordering. CASCADE on
PRESERVED: fandom + series tags and their series_page ordering, AND the
system hygiene tags (#128) WITH their applications — the reset re-tags
CONTENT concepts, while wip/banner flags describe the file itself and
re-flagging hundreds of banners by hand would be pure loss. CASCADE on
image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
applications + metadata. Tag.fandom_id is SET NULL, so deleting character
tags never touches the fandom rows. Irreversible except via DB backup
@@ -744,7 +752,9 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
"sample_names": [first 50],
and on live runs "deleted": total}
"""
predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
predicate = and_(
Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False)
)
rows = session.execute(
select(Tag.id, Tag.name, Tag.kind).where(predicate)
).all()
+15 -1
View File
@@ -23,7 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import aliased
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
from ..models.tag import image_tag
from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag
from .pagination import decode_cursor, encode_cursor
from .tag_query import (
fandom_join_alias,
@@ -693,9 +693,23 @@ class GalleryService:
eff = _effective_date_col()
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
stmt = _outer_join_primary_post(stmt)
# Presentation images (banner / editor-screenshot system tags, #128)
# cluster on UI chrome rather than content, so near any one of them
# they'd fill the grid. Excluded from CANDIDATES only — the anchor
# itself may be a banner, and `wip` stays surfaced (real art; only
# the training pipelines exclude it).
presentation = (
select(image_tag.c.image_record_id)
.join(Tag, Tag.id == image_tag.c.tag_id)
.where(
Tag.is_system.is_(True),
Tag.name.in_(PRESENTATION_SYSTEM_TAGS),
)
)
stmt = stmt.where(
ImageRecord.siglip_embedding.is_not(None),
ImageRecord.id != image_id,
ImageRecord.id.not_in(presentation),
)
stmt = _apply_scope(
stmt, tag_ids=tag_ids, post_id=None,
+26 -1
View File
@@ -62,6 +62,18 @@ def _single_character_images():
)
def _hygiene_tagged_images():
"""Subquery of image ids carrying any SYSTEM tag (wip / banner / editor
screenshot). Training hygiene (#128): such images never contribute
reference prototypes — a faceless wip's figure region would otherwise
become an identity reference for the character it's tagged with."""
return (
select(image_tag.c.image_record_id)
.join(Tag, Tag.id == image_tag.c.tag_id)
.where(Tag.is_system.is_(True))
)
async def _ref_signature(session: AsyncSession) -> tuple:
n_tags = (
await session.execute(
@@ -79,7 +91,17 @@ async def _ref_signature(session: AsyncSession) -> tuple:
)
)
).one()
return (n_tags, n_regs, max_id)
# Hygiene applications must invalidate too: tagging an image `wip` changes
# the reference set without touching character-tag or region counts.
n_hygiene = (
await session.execute(
select(func.count())
.select_from(image_tag)
.join(Tag, Tag.id == image_tag.c.tag_id)
.where(Tag.is_system.is_(True))
)
).scalar_one()
return (n_tags, n_regs, max_id, n_hygiene)
async def character_references(session: AsyncSession) -> dict[int, list]:
@@ -102,6 +124,9 @@ async def character_references(session: AsyncSession) -> dict[int, list]:
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
.where(ImageRegion.ccip_embedding.is_not(None))
.where(ImageRegion.image_record_id.in_(_single_character_images()))
.where(
ImageRegion.image_record_id.not_in(_hygiene_tagged_images())
)
)
).all()
refs: dict[int, list] = {}
+48 -13
View File
@@ -40,6 +40,7 @@ from ...models import (
from ...models.tag import image_tag
from .training_data import (
_auto_apply_point,
_hygiene_excluded_ids,
_ids_with_tag,
_l2norm,
_load_embeddings,
@@ -150,11 +151,16 @@ def train_all_heads(
embedding_version = _embedder_version(session)
eligible = _eligible_tag_ids(session, cfg["min_positives"])
eligible_set = set(eligible)
# Computed once per run, not per head — the hygiene set is identical for
# every non-system concept.
hygiene = _hygiene_excluded_ids(session)
trained = 0
skipped = 0
for i, tag_id in enumerate(eligible):
try:
ok = train_head(session, tag_id, embedding_version, cfg, np)
ok = train_head(
session, tag_id, embedding_version, cfg, np, hygiene=hygiene
)
except Exception:
log.exception("train_head failed for tag %d", tag_id)
ok = False
@@ -174,27 +180,56 @@ def train_all_heads(
return {"n_trained": trained, "n_skipped": skipped}
def head_training_ids(
session: Session, tag_id: int, cfg: dict, hygiene: set[int] | None = None,
) -> tuple[list[int], list[int]] | None:
"""Select (pos_ids, neg_ids) for one head. Split out of train_head and
kept sklearn-free so the hygiene exclusion is testable in the CI env
(sklearn only exists in the ml image). Returns None when the concept has
too few usable positives.
Training hygiene (#128): images carrying a system tag are ABSENT from
every other concept's training — dropped as positives AND kept out of
the rejection/sampled negative pool (see _hygiene_excluded_ids). A system
tag's own head trains on them unfiltered: its positives ARE the hygiene
images."""
tag = session.get(Tag, tag_id)
if tag is not None and tag.is_system:
hygiene = set()
elif hygiene is None:
hygiene = _hygiene_excluded_ids(session)
pos_ids = [i for i in _ids_with_tag(session, tag_id) if i not in hygiene]
if len(pos_ids) < cfg["min_positives"]:
return None
pos_set = set(pos_ids)
rejected = [
i for i in _rejected_ids(session, tag_id)
if i not in pos_set and i not in hygiene
]
want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
sampled = _sample_unlabeled(
session, pos_set | set(rejected) | hygiene,
min(_UNLABELED_POOL, want_neg),
)
return pos_ids, rejected + [i for i in sampled if i not in pos_set]
def train_head(
session: Session, tag_id: int, embedding_version: str, cfg: dict, np
session: Session, tag_id: int, embedding_version: str, cfg: dict, np,
hygiene: set[int] | None = None,
) -> bool:
"""Fit + upsert one head. Returns True if a head was written, False if the
concept had too few usable examples to train (the row is then removed)."""
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_predict
pos_ids = _ids_with_tag(session, tag_id)
if len(pos_ids) < cfg["min_positives"]:
ids = head_training_ids(session, tag_id, cfg, hygiene)
if ids is None:
session.execute(delete(TagHead).where(TagHead.tag_id == tag_id))
return False
pos_set = set(pos_ids)
rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set]
want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
sampled = _sample_unlabeled(
session, pos_set | set(rejected), min(_UNLABELED_POOL, want_neg)
)
neg_ids = rejected + [i for i in sampled if i not in pos_set]
pos_ids, neg_ids = ids
emb = _load_embeddings(session, pos_ids + neg_ids)
pos = [emb[i] for i in pos_ids if i in emb]
neg = [emb[i] for i in neg_ids if i in emb]
+24 -1
View File
@@ -17,10 +17,33 @@ from typing import Any
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from ...models import ImageRecord, TagSuggestionRejection
from ...models import ImageRecord, Tag, TagSuggestionRejection
from ...models.tag import image_tag
def _hygiene_excluded_ids(session: Session) -> set[int]:
"""Ids of images carrying ANY system tag (wip / banner / editor
screenshot — milestone #128). These images are excluded from OTHER
concepts' head training entirely: not positives (a rough wip tagged as a
character drags that head toward 'generic sketch') and not sampled or
rejection negatives (a wip OF character X is not evidence against X) —
simply absent. A system tag's OWN head trains on them unchanged; that is
what makes auto-flagging banners/editor screenshots work.
Item-level by design: a wip-tagged process video contributes (or
withholds) ALL its sampled frames, though some may show the finished
piece. Operator call 2026-07-03: with enough clean data this washes out —
no per-frame handling.
"""
return set(
session.execute(
select(image_tag.c.image_record_id)
.join(Tag, Tag.id == image_tag.c.tag_id)
.where(Tag.is_system.is_(True))
).scalars().all()
)
def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
return [
r[0] for r in session.execute(
+25 -11
View File
@@ -14,9 +14,11 @@
`_personalization_id` age-confirmation cookie that the user can't
easily refresh — SubscribeStar's frontend JS uses localStorage to
suppress the age popup once dismissed. gallery-dl's own login flow
sidesteps this by setting `18_plus_agreement_generic=true` on
`.subscribestar.adult`; we mirror that for cookies captured via the
extension.
sidesteps this by setting `18_plus_agreement_generic=true`; we mirror
that for cookies captured via the extension — on EVERY SubscribeStar
domain, because cookies are domain-scoped and the wall exists on all
of them: an .adult-only line never rides along to a .art creator page
(Elasid, event #54116 — every poll 302'd to /age_confirmation_warning).
"""
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
@@ -29,20 +31,31 @@ def derive_post_url(data: dict) -> str | None:
return None
# All domains SubscribeStar serves creators from; the age wall gates each.
_SS_DOMAINS = (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art")
def augment_cookies(netscape: str) -> str:
if "18_plus_agreement_generic" in netscape:
return netscape
# Far-future expiry — gallery-dl's own login flow sets this with no
# explicit expiry; the server only checks presence/value.
expiry = 4102444800 # 2100-01-01 UTC
line = "\t".join([
".subscribestar.adult", "TRUE", "/", "TRUE",
str(expiry), "18_plus_agreement_generic", "true",
])
body = netscape.rstrip("\n")
if not body:
body = "# Netscape HTTP Cookie File"
return body + "\n" + line + "\n"
lines = netscape.splitlines()
for domain in _SS_DOMAINS:
# Per-domain presence check: captured cookies carrying the age
# cookie for one domain must not suppress injection on the others.
if any(
line.startswith(domain + "\t") and "18_plus_agreement_generic" in line
for line in lines
):
continue
body += "\n" + "\t".join([
domain, "TRUE", "/", "TRUE",
str(expiry), "18_plus_agreement_generic", "true",
])
return body + "\n"
INFO = PlatformInfo(
@@ -51,10 +64,11 @@ INFO = PlatformInfo(
description="Download posts from SubscribeStar creators",
auth_type="cookies",
requires_auth=True,
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/",
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult|art)/",
url_examples=[
"https://subscribestar.adult/example_artist",
"https://www.subscribestar.com/example_artist",
"https://subscribestar.art/example_artist",
],
default_config={**GD_DEFAULTS, "content_types": ["all"]},
derive_post_url=derive_post_url,
@@ -107,6 +107,7 @@ class TagDirectoryService:
"kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind,
"fandom_id": tag.fandom_id,
"fandom_name": fandom_name,
"is_system": tag.is_system,
"image_count": int(image_count),
"preview_thumbnails": previews.get(tag.id, []),
}
+7 -3
View File
@@ -67,24 +67,28 @@ def fandom_join_alias():
def tag_columns(fandom_alias):
"""The canonical (id, name, kind, fandom_id, fandom_name) column set for a
tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`)."""
"""The canonical (id, name, kind, fandom_id, fandom_name, is_system)
column set for a tag select that outerjoins `fandom_alias` (from
`fandom_join_alias()`)."""
return [
Tag.id,
Tag.name,
Tag.kind,
Tag.fandom_id,
fandom_alias.c.name.label("fandom_name"),
Tag.is_system,
]
def serialize_tag(row) -> dict:
"""Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the
canonical tag dict. `kind` may be a TagKind enum or a plain string."""
canonical tag dict. `kind` may be a TagKind enum or a plain string.
`is_system` defaults False for callers whose selects predate the column."""
return {
"id": row.id,
"name": row.name,
"kind": row.kind.value if hasattr(row.kind, "value") else row.kind,
"fandom_id": row.fandom_id,
"fandom_name": row.fandom_name,
"is_system": bool(getattr(row, "is_system", False)),
}
+18
View File
@@ -330,6 +330,12 @@ class TagService:
tag = await self.session.get(Tag, tag_id)
if tag is None:
raise TagValidationError(f"Tag {tag_id} not found")
# The training-hygiene machinery keys on system tags by row; a rename
# would silently disarm it (milestone #128).
if tag.is_system:
raise TagValidationError(
f"'{tag.name}' is a system tag and cannot be renamed"
)
# Case-insensitive clash (#701) — renaming onto a differently-cased tag
# is still a merge, not a silent fork.
@@ -528,6 +534,14 @@ class TagService:
source = await self.session.get(Tag, source_id)
if source is None:
raise TagValidationError(f"Tag {source_id} not found")
# Merge deletes the source row — the only tag-delete path — and the
# training-hygiene machinery keys on system rows (milestone #128).
# Merging INTO a system tag stays allowed (adopting an operator's old
# 'wip sketch' tag into the system 'wip' is the intended move).
if source.is_system:
raise TagValidationError(
f"'{source.name}' is a system tag and cannot be merged away"
)
target = await self.session.get(Tag, target_id)
if target is None:
raise TagValidationError(f"Tag {target_id} not found")
@@ -792,6 +806,10 @@ async def normalize_existing_tags(
rows = (
await session.execute(
select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
# System tags (#128) are exempt: their names are the keys the
# hygiene machinery matches on, and canonicalization would recase
# 'wip' → 'Wip' (this path bypasses TagService.rename's guard).
.where(Tag.is_system.is_(False))
)
).all()
groups = _group_existing_tags(rows)
@@ -13,9 +13,15 @@
<div class="fc-tagcard__name">
<template v-if="!editing">
{{ card.name }}
<v-icon
v-if="card.is_system" size="14" icon="mdi-shield-outline"
class="fc-tagcard__system"
title="System tag — tagged items are excluded from training other concepts"
/>
<span v-if="card.kind === 'character' && card.fandom_name"
class="fc-tagcard__fandom"> {{ card.fandom_name }}</span>
<v-icon
v-if="!card.is_system"
class="fc-tagcard__edit" size="14" icon="mdi-pencil"
@click.stop="startEdit"
/>
@@ -58,12 +64,18 @@
prepend-icon="mdi-tag-multiple"
@click="$emit('aliases', card)"
/>
<!-- System tags (#128): merge-away and delete are refused
server-side the hygiene machinery keys on the row so
don't offer them. Aliases stay (mapping model outputs ONTO
a system tag is useful). -->
<v-list-item
v-if="!card.is_system"
title="Merge with…"
prepend-icon="mdi-call-merge"
@click="$emit('merge-with', card)"
/>
<v-list-item
v-if="!card.is_system"
title="Delete tag"
prepend-icon="mdi-delete"
base-color="error"
@@ -133,6 +145,7 @@ function submit() {
.fc-tagcard__body { padding: 8px 12px; }
.fc-tagcard__name { font-weight: 600; }
.fc-tagcard__fandom { font-weight: 400; opacity: 0.7; }
.fc-tagcard__system { opacity: 0.6; margin-left: 2px; }
.fc-tagcard__meta {
display: flex; align-items: center; justify-content: space-between;
margin-top: 4px;
@@ -100,6 +100,13 @@ import { useSuggestionsStore } from '../../stores/suggestions.js'
import FandomPicker from './FandomPicker.vue'
const emit = defineEmits(['pick-existing', 'pick-new', 'accept-suggestion', 'cancel'])
// The host surface's applied tags (host.current?.tags). Both dropdown sections
// filter against this so a just-added tag drops out of the search the moment
// the chip rail updates, not after the next modal refresh (operator-asked
// 2026-07-03).
const props = defineProps({
appliedTags: { type: Array, default: () => [] },
})
const store = useTagStore()
// The image's ML (Camie) suggestions, surfaced inline in this dropdown so the
// operator can pick a suggestion while typing instead of hunting for it in the
@@ -208,6 +215,18 @@ const createLabel = computed(() =>
function scorePct (s) { return `${Math.round(s.score * 100)}%` }
const appliedIds = computed(() => new Set(props.appliedTags.map(t => t.id)))
const appliedNames = computed(() =>
new Set(props.appliedTags.map(t => `${t.kind}:${(t.name || '').toLowerCase()}`)),
)
// Server matches minus the image's applied tags. allowCreate/sameNameCharExists
// keep reading the RAW hits — they reason about the tag universe (does this tag
// exist?), not about this image.
const visibleHits = computed(() =>
hits.value.filter(h => !appliedIds.value.has(h.id)),
)
// This image's suggestions that match the typed query, minus any the server
// autocomplete already returned (same name+kind) so a tag never shows twice.
// Sources the FULL prediction set (allByCategory, down to the store floor) — NOT
@@ -218,7 +237,7 @@ function scorePct (s) { return `${Math.round(s.score * 100)}%` }
const suggestionHits = computed(() => {
const q = parsedName.value.toLowerCase()
if (!q) return []
const seen = new Set(hits.value.map(h => `${h.kind}:${h.name.toLowerCase()}`))
const seen = new Set(visibleHits.value.map(h => `${h.kind}:${h.name.toLowerCase()}`))
const out = []
for (const list of Object.values(suggestions.allByCategory)) {
for (const s of list || []) {
@@ -226,7 +245,12 @@ const suggestionHits = computed(() => {
// can show + un-reject them; keep them OUT of the type-to-add dropdown,
// whose job is finding a tag to ADD (un-reject lives in the panel).
if (s.rejected) continue
// Already on the image (matched by canonical id or name+kind): nothing
// to add. Covers the window between an add and the next suggestions
// fetch, where the stale row would otherwise still be pickable.
if (s.canonical_tag_id != null && appliedIds.value.has(s.canonical_tag_id)) continue
const key = `${s.category}:${s.display_name.toLowerCase()}`
if (appliedNames.value.has(key)) continue
if (!s.display_name.toLowerCase().includes(q)) continue
if (seen.has(key)) continue
seen.add(key)
@@ -242,7 +266,7 @@ const suggestionHits = computed(() => {
// One ordered list backing both the rendered dropdown and keyboard nav, so the
// highlight index maps 1:1 to a row regardless of which section it's in.
const rows = computed(() => {
const r = hits.value.map(h => ({ type: 'hit', key: `h${h.id}`, hit: h }))
const r = visibleHits.value.map(h => ({ type: 'hit', key: `h${h.id}`, hit: h }))
for (const s of suggestionHits.value) {
r.push({ type: 'suggestion', key: `s${s.category}:${s.display_name}`, sugg: s })
}
+13 -3
View File
@@ -10,14 +10,23 @@
@click:close="$emit('remove', tag.id)"
>
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
{{ tag.name }}<span
{{ tag.name }}<v-icon
v-if="tag.is_system" end size="x-small" class="fc-tag-chip__system"
title="System tag — tagged items are excluded from training other concepts"
>mdi-shield-outline</v-icon><span
v-if="tag.fandom_id" class="fc-tag-chip__fandom"
:title="tag.fandom_name ? `Fandom: ${tag.fandom_name}` : 'Has a fandom'"
><template v-if="fandomLabel">&nbsp;{{ fandomLabel }}</template></span>
</v-chip>
<!-- Modal-safe kebab is baked into KebabMenu (this chip lives in the
teleported image modal #711). -->
<KebabMenu class="fc-tag-chip__kebab" :label="`More actions for ${tag.name}`">
teleported image modal #711). System tags hide it entirely: rename
is refused server-side (the hygiene machinery keys on the row) and
set-fandom never applies to their general kind. Remove () stays
un-tagging an image is normal use. -->
<KebabMenu
v-if="!tag.is_system"
class="fc-tag-chip__kebab" :label="`More actions for ${tag.name}`"
>
<v-list-item @click="$emit('rename', tag)">
<v-list-item-title>Rename</v-list-item-title>
</v-list-item>
@@ -64,6 +73,7 @@ function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
outline: 2px solid rgb(var(--v-theme-accent));
outline-offset: 1px;
}
.fc-tag-chip__system { opacity: 0.6; margin-left: 2px; }
.fc-tag-chip__kebab { opacity: 0.7; }
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
.fc-tag-chip__fandom { opacity: 0.7; font-size: 0.85em; }
+43 -3
View File
@@ -15,6 +15,7 @@
<TagAutocomplete
ref="tagInputRef"
:applied-tags="host.current?.tags || []"
@pick-existing="onPickExisting" @pick-new="onPickNew"
@accept-suggestion="onAcceptSuggestion"
/>
@@ -107,17 +108,56 @@ defineExpose({ focusTagInput })
// (operator-asked 2026-06-26; the Explore workspace leans on this hard).
async function onRemove(tagId) {
errorMsg.value = null
try { await host.removeTag(tagId); focusTagInput() }
try {
await host.removeTag(tagId)
// removeTag also records a per-image dismissal server-side, so reloading
// puts a model-suggested tag BACK in the suggestions area (flagged
// rejected — visible + one-click reversible) instead of it vanishing
// until the next modal open (operator-asked 2026-07-03).
if (host.currentImageId != null) {
suggestions.load(host.currentImageId)
suggestions.loadAll(host.currentImageId)
}
focusTagInput()
}
catch (e) { errorMsg.value = e.message }
}
async function onPickExisting(hit) {
errorMsg.value = null
try { await host.addExistingTag(hit.id); focusTagInput() }
try {
// Manually picking a tag the model also suggested = accepting the
// suggestion: the acceptance is recorded and the row drops from the
// panel, instead of silently bypassing the feedback loop
// (operator-asked 2026-07-03).
const pending = suggestions.findPending(hit.kind, hit.name, hit.id)
if (pending) {
await suggestions.accept(pending, { tagId: hit.id })
await host.reloadTags()
} else {
await host.addExistingTag(hit.id)
}
focusTagInput()
}
catch (e) { errorMsg.value = e.message }
}
async function onPickNew(payload) {
errorMsg.value = null
try { await host.createAndAdd(payload); focusTagInput() }
try {
const imageId = host.currentImageId
const created = await host.createAndAdd(payload)
// The typed name can match a raw (or canonical) suggestion; creating it
// by hand still counts as accepting — record it and drop the row. Runs
// AFTER createAndAdd so a character's fandom pick is preserved (accept's
// own create path knows no fandom); the accept apply is idempotent.
// Guard on the image not having changed mid-create (fast prev/next).
if (created && host.currentImageId === imageId) {
const pending = suggestions.findPending(payload.kind, payload.name, created.id)
if (pending) {
await suggestions.accept(pending, { tagId: created.id })
}
}
focusTagInput()
}
catch (e) { errorMsg.value = e.message }
}
// A suggestion picked from the autocomplete dropdown runs the SAME path as the
+3
View File
@@ -176,6 +176,9 @@ export const useExploreStore = defineStore('explore', () => {
}
if (anchor.value?.id !== id) return // walked away mid-create
await addExistingTag(tag.id)
// Returned so TagPanel can match the created tag against a pending
// suggestion and record the acceptance (manual add == accept, 2026-07-03).
return tag
}
// No overlay to dismiss in the Explore workspace — the chip-body "show me
+3
View File
@@ -170,6 +170,9 @@ export const useModalStore = defineStore('modal', () => {
}
if (currentImageId.value !== imageId) return // navigated away
await addExistingTag(tag.id)
// Returned so TagPanel can match the created tag against a pending
// suggestion and record the acceptance (manual add == accept, 2026-07-03).
return tag
}
const isOpen = computed(() => currentImageId.value !== null)
+32 -3
View File
@@ -97,7 +97,12 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
}
}
async function accept(suggestion) {
// opts.tagId: the tag's known id, when the caller already created/resolved
// it (TagPanel's manual-add-matches-suggestion path). Passed separately —
// NOT spread onto the suggestion — because _dropEverywhere keys off the
// suggestion object, and mutating canonical_tag_id on a raw suggestion
// would stop it matching its own row in the lists.
async function accept(suggestion, { tagId: knownTagId = null } = {}) {
// Capture imageId so a mid-flight prev/next can't reroute the
// accept POST to a different image AND push the tag to that
// image's allowlist.
@@ -106,7 +111,7 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
// Raw tags (creates_new_tag) have no canonical_tag_id; the backend's
// accept endpoint needs a tag_id, so for raw tags we create the tag
// first via the existing /api/tags endpoint, then accept by id.
let tagId = suggestion.canonical_tag_id
let tagId = knownTagId ?? suggestion.canonical_tag_id
if (tagId == null) {
const created = await api.post('/api/tags', {
body: { name: suggestion.display_name, kind: suggestion.category }
@@ -169,6 +174,29 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
toast({ text: `Alias removed: ${suggestion.display_name}`, type: 'success' })
}
// Find a live (non-rejected) pending suggestion matching a manually-picked
// tag — by canonical id when known, else by (kind, name). Manually adding a
// tag the model also suggested must be indistinguishable from accepting the
// suggestion (recorded + dropped from the panel), so TagPanel routes matches
// through accept() (operator-asked 2026-07-03). Searches the full dropdown
// set first, then the panel list (loadAll is best-effort and can be empty).
function findPending(kind, name, tagId = null) {
const lname = (name || '').toLowerCase()
for (const map of [allByCategory.value, byCategory.value]) {
for (const list of Object.values(map)) {
for (const s of list || []) {
if (s.rejected) continue
if (tagId != null && s.canonical_tag_id === tagId) return s
if (
s.category === kind &&
(s.display_name || '').toLowerCase() === lname
) return s
}
}
}
return null
}
async function dismiss(suggestion) {
const imageId = currentImageId
if (imageId == null) return
@@ -202,6 +230,7 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
return {
byCategory, allByCategory, loading, error,
load, loadAll, accept, aliasAccept, removeAlias, dismiss, undismiss
load, loadAll, accept, aliasAccept, removeAlias, dismiss, undismiss,
findPending
}
})
+112
View File
@@ -0,0 +1,112 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { setActivePinia, createPinia } from 'pinia'
import { useSuggestionsStore } from '../src/stores/suggestions.js'
vi.mock('../src/utils/toast.js', () => ({ toast: vi.fn() }))
function stubFetch(handler) {
globalThis.fetch = vi.fn(async (url, init) => {
const { status, body } = handler(url, init)
return {
ok: status >= 200 && status < 300,
status,
statusText: String(status),
text: async () => (body == null ? '' : JSON.stringify(body)),
}
})
}
const sugg = (over = {}) => ({
canonical_tag_id: 7,
display_name: 'Ichigo',
category: 'character',
score: 0.9,
source: 'head',
creates_new_tag: false,
rejected: false,
...over,
})
// Seed the store through its own load/loadAll so currentImageId is set the
// way the panel sets it (accept() derives the image from it).
async function seed(store, byCategory) {
stubFetch(() => ({ status: 200, body: { by_category: byCategory } }))
await store.load(1)
await store.loadAll(1)
}
describe('suggestions store: findPending (manual add == accept, 2026-07-03)', () => {
beforeEach(() => setActivePinia(createPinia()))
afterEach(() => vi.restoreAllMocks())
it('matches by canonical tag id', async () => {
const s = useSuggestionsStore()
await seed(s, { character: [sugg()] })
expect(s.findPending('general', 'unrelated', 7)?.display_name).toBe('Ichigo')
})
it('matches raw suggestions by kind + name, case-insensitively', async () => {
const s = useSuggestionsStore()
await seed(s, {
general: [sugg({
canonical_tag_id: null, display_name: 'Holding Sword',
category: 'general', creates_new_tag: true,
})],
})
expect(s.findPending('general', 'holding sword')).toBeTruthy()
// Kind must match: same name under another kind is a different tag.
expect(s.findPending('character', 'holding sword')).toBeNull()
})
it('skips rejected rows and returns null when nothing matches', async () => {
const s = useSuggestionsStore()
await seed(s, { character: [sugg({ rejected: true })] })
expect(s.findPending('character', 'Ichigo', 7)).toBeNull()
expect(s.findPending('character', 'Rukia')).toBeNull()
})
})
describe('suggestions store: accept with a known tag id', () => {
beforeEach(() => setActivePinia(createPinia()))
afterEach(() => vi.restoreAllMocks())
it('POSTs only the accept endpoint (no tag create) and drops the row', async () => {
const s = useSuggestionsStore()
await seed(s, { character: [sugg()] })
const calls = []
stubFetch((url, init) => {
calls.push({ url, method: init?.method })
return { status: 200, body: { accepted: true, tag_id: 7 } }
})
await s.accept(s.byCategory.character[0], { tagId: 7 })
expect(calls).toHaveLength(1)
expect(calls[0].url).toContain('/api/images/1/suggestions/accept')
expect(s.byCategory.character).toHaveLength(0)
expect(s.allByCategory.character).toHaveLength(0)
})
it('a RAW suggestion accepted with a known tagId skips create AND still drops its row', async () => {
// TagPanel's manual-create path: the tag was just created by
// host.createAndAdd, so accept must not create again — and the drop must
// key off the original raw suggestion object, not the resolved id
// (regression: spreading canonical_tag_id onto the suggestion changed its
// identity key and left the panel row behind).
const s = useSuggestionsStore()
const raw = sugg({
canonical_tag_id: null, display_name: 'Holding Sword',
category: 'general', creates_new_tag: true,
})
await seed(s, { general: [raw] })
const calls = []
stubFetch((url, init) => {
calls.push({ url, body: init?.body })
return { status: 200, body: { accepted: true, tag_id: 42 } }
})
await s.accept(s.byCategory.general[0], { tagId: 42 })
expect(calls).toHaveLength(1)
expect(calls[0].url).toContain('/api/images/1/suggestions/accept')
expect(JSON.parse(calls[0].body)).toEqual({ tag_id: 42 })
expect(s.byCategory.general).toHaveLength(0)
expect(s.allByCategory.general).toHaveLength(0)
})
})
+11
View File
@@ -166,6 +166,17 @@ def _reset_db_after_integration(request, _truncate_engine):
rows = (_SEED_SNAPSHOT or {}).get(t.name)
if rows:
conn.execute(t.insert(), rows)
# Restored rows carry their explicit ids while RESTART
# IDENTITY reset the sequence to 1, so the next ORM insert
# would collide on the PK (bitten by migration 0075's seeded
# system tags: every Tag insert failed with pk_tag id=1).
# Resync any serial id sequence past the restored rows.
if "id" in t.c:
conn.exec_driver_sql(
f"SELECT setval(pg_get_serial_sequence('{t.name}', 'id'), "
f"(SELECT COALESCE(MAX(id), 1) FROM {t.name})) "
f"WHERE pg_get_serial_sequence('{t.name}', 'id') IS NOT NULL"
)
@pytest_asyncio.fixture(autouse=True)
+3 -1
View File
@@ -660,8 +660,10 @@ async def test_reset_content_tagging_apply_requires_confirm_token(client, db):
for bad in ({"dry_run": False}, {"dry_run": False, "confirm": "deadbeef"}):
resp = await client.post("/api/admin/tags/reset-content", json=bad)
assert resp.status_code == 400
# Non-system only: the seeded hygiene tags (#128) survive a reset by
# design, so they'd inflate a whole-table count.
remaining = (await db.execute(
select(func.count()).select_from(Tag)
select(func.count()).select_from(Tag).where(Tag.is_system.is_(False))
)).scalar_one()
assert remaining == 1
+15 -6
View File
@@ -377,8 +377,11 @@ def test_prune_unused_tags_dry_run_returns_count_and_names(db_sync, tmp_path):
assert result["count"] == 2
assert "prune-me-1" in result["sample_names"]
assert "prune-me-2" in result["sample_names"]
# Nothing actually deleted.
surviving = db_sync.execute(select(func.count(Tag.id))).scalar_one()
# Nothing actually deleted. Non-system only — the seeded hygiene tags
# (#128) are exempt from pruning and would inflate a whole-table count.
surviving = db_sync.execute(
select(func.count(Tag.id)).where(Tag.is_system.is_(False))
).scalar_one()
assert surviving == 3
@@ -499,8 +502,11 @@ def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path
assert result["count"] == 2
assert result["by_kind"] == {"general": 1, "character": 1}
assert result["applications"] == 2
# Nothing deleted — all 4 tags still present.
assert db_sync.execute(select(func.count(Tag.id))).scalar_one() == 4
# Nothing deleted — all 4 fixture tags still present (non-system count;
# the seeded hygiene tags #128 also survive, by design).
assert db_sync.execute(
select(func.count(Tag.id)).where(Tag.is_system.is_(False))
).scalar_one() == 4
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
@@ -525,8 +531,11 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
assert result["deleted"] == 2
# general + character gone; fandom + series kept.
kinds = db_sync.execute(select(Tag.kind)).scalars().all()
# general + character gone; fandom + series kept. Non-system only — the
# seeded hygiene tags (#128, kind=general) survive a reset by design.
kinds = db_sync.execute(
select(Tag.kind).where(Tag.is_system.is_(False))
).scalars().all()
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
assert kind_vals == {"fandom", "series"}
# series_page ordering survived.
+20 -8
View File
@@ -116,8 +116,10 @@ async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp
cookie; the browser-stored cookie expires annually and can't be easily
refreshed (the JS age popup is suppressed by localStorage). Mirror
gallery-dl's own login-flow workaround by injecting
`18_plus_agreement_generic=true` on `.subscribestar.adult` whenever
cookies for subscribestar are materialized."""
`18_plus_agreement_generic=true` whenever cookies for subscribestar
are materialized — on EVERY SubscribeStar domain, since cookies are
domain-scoped and creators live on .com/.adult/.art alike (a .adult-only
line left .art sources stuck on the age wall — Elasid, event #54116)."""
netscape_in = (
"# Netscape HTTP Cookie File\n"
".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n"
@@ -126,16 +128,18 @@ async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp
await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in)
path = await svc.get_cookies_path("subscribestar")
contents = path.read_text()
assert "18_plus_agreement_generic\ttrue" in contents
assert ".subscribestar.adult" in contents
for domain in (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art"):
assert f"{domain}\tTRUE\t/\tTRUE\t4102444800\t18_plus_agreement_generic\ttrue" in contents
# Original session_id cookie preserved.
assert "session_id\txyz" in contents
@pytest.mark.asyncio
async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto, tmp_path):
"""If the operator's captured cookies ALREADY contain the age cookie
(e.g. a manual paste, or a re-login), don't double-inject."""
async def test_get_cookies_path_subscribestar_idempotent_per_domain(db, crypto, tmp_path):
"""If the operator's captured cookies ALREADY contain the age cookie for
a domain (e.g. a manual paste, or a re-login), don't double-inject on
THAT domain — but a .adult-only capture must still get the cookie added
for .com and .art (per-domain check, not presence-anywhere)."""
netscape_in = (
"# Netscape HTTP Cookie File\n"
".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\t18_plus_agreement_generic\ttrue\n"
@@ -145,7 +149,15 @@ async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto
await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in)
path = await svc.get_cookies_path("subscribestar")
contents = path.read_text()
assert contents.count("18_plus_agreement_generic") == 1
assert contents.count("18_plus_agreement_generic") == 3
adult_lines = [
line for line in contents.splitlines()
if line.startswith(".subscribestar.adult\t")
and "18_plus_agreement_generic" in line
]
assert len(adult_lines) == 1
assert ".subscribestar.com\t" in contents
assert ".subscribestar.art\t" in contents
@pytest.mark.asyncio
+26
View File
@@ -4,6 +4,7 @@ precomputed SigLIP image embeddings. No query-time ML inference."""
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import select
from backend.app.models import ImageRecord, Tag, TagKind
from backend.app.models.tag import image_tag
@@ -72,6 +73,31 @@ async def test_similar_missing_source_returns_none(db):
assert await svc.similar(99999, limit=10) is None
@pytest.mark.asyncio
async def test_similar_excludes_presentation_tagged_images(db):
"""banner / editor-screenshot system tags (#128) hide from similar
RESULTS — they cluster on UI chrome, not content. A banner anchor still
gets results; `wip`-tagged images stay in (real art)."""
src = await _img(db, 1, _vec(1, 0))
bannered = await _img(db, 2, _vec(1, 0.02)) # nearest, but a banner
wipped = await _img(db, 3, _vec(1, 0.3))
plain = await _img(db, 4, _vec(1, 0.6))
banner_tag = (await db.execute(select(Tag).where(
Tag.is_system.is_(True), Tag.name == "banner"))).scalar_one()
wip_tag = (await db.execute(select(Tag).where(
Tag.is_system.is_(True), Tag.name == "wip"))).scalar_one()
await db.execute(image_tag.insert().values(
image_record_id=bannered.id, tag_id=banner_tag.id, source="manual"))
await db.execute(image_tag.insert().values(
image_record_id=wipped.id, tag_id=wip_tag.id, source="manual"))
svc = GalleryService(db)
res = await svc.similar(src.id, limit=10)
assert [i.id for i in res] == [wipped.id, plain.id]
# A presentation-tagged ANCHOR still answers — only candidates hide.
res_from_banner = await svc.similar(bannered.id, limit=10)
assert {i.id for i in res_from_banner} == {src.id, wipped.id, plain.id}
@pytest.mark.asyncio
async def test_similar_composes_with_tag_filter(db):
src = await _img(db, 1, _vec(1, 0))
+152
View File
@@ -0,0 +1,152 @@
"""System tags (milestone #128): seeded rows, protection guards, serialization.
The three hygiene tags (wip / banner / editor screenshot) ship via migration
0075 and drive the training-hygiene exclusions. They must exist after
migration, serialize with is_system, and refuse rename/merge-away — the
mechanism keys on these exact rows. Merge-INTO stays allowed: adopting an
operator's old hygiene tag into the system row is the intended move.
"""
import pytest
from sqlalchemy import insert, select
from backend.app.models import ImageRecord, Tag, TagKind
from backend.app.models.tag import image_tag
from backend.app.services.cleanup_service import (
prune_unused_tags,
reset_content_tagging,
)
pytestmark = pytest.mark.integration
SYSTEM_NAMES = {"wip", "banner", "editor screenshot"}
async def _system_tag(db, name):
return (await db.execute(
select(Tag).where(Tag.is_system.is_(True), Tag.name == name)
)).scalar_one()
@pytest.mark.asyncio
async def test_seeded_system_tags_exist(db):
rows = (await db.execute(
select(Tag.name, Tag.kind).where(Tag.is_system.is_(True))
)).all()
assert {r.name for r in rows} == SYSTEM_NAMES
assert all(r.kind == TagKind.general for r in rows)
@pytest.mark.asyncio
async def test_get_tag_serializes_is_system(client, db):
wip = await _system_tag(db, "wip")
resp = await client.get(f"/api/tags/{wip.id}")
assert resp.status_code == 200
assert (await resp.get_json())["is_system"] is True
@pytest.mark.asyncio
async def test_rename_system_tag_refused(client, db):
wip = await _system_tag(db, "wip")
resp = await client.patch(
f"/api/tags/{wip.id}", json={"name": "work in progress"}
)
assert resp.status_code == 400
body = await resp.get_json()
assert "system tag" in body["error"]
@pytest.mark.asyncio
async def test_merge_away_system_tag_refused(client, db):
banner = await _system_tag(db, "banner")
created = await client.post(
"/api/tags", json={"name": "ad banner", "kind": "general"}
)
target_id = (await created.get_json())["id"]
resp = await client.post(
f"/api/tags/{banner.id}/merge", json={"target_id": target_id}
)
assert resp.status_code == 400
body = await resp.get_json()
assert "system tag" in body["error"]
@pytest.mark.asyncio
async def test_admin_delete_system_tag_refused(client, db):
wip = await _system_tag(db, "wip")
resp = await client.delete(f"/api/admin/tags/{wip.id}")
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "system_tag"
still = (await db.execute(
select(Tag.id).where(Tag.id == wip.id)
)).scalar_one_or_none()
assert still == wip.id
@pytest.mark.asyncio
async def test_prune_unused_skips_system_tags(db):
"""The seeded system tags have zero applications and would match every
'unused' condition — the shared predicate must exempt them (preview AND
live delete, same conditions)."""
preview = await db.run_sync(
lambda s: prune_unused_tags(s, dry_run=True)
)
assert preview["count"] == 0
assert preview["sample_names"] == []
await db.run_sync(lambda s: prune_unused_tags(s, dry_run=False))
remaining = (await db.execute(
select(Tag.name).where(Tag.is_system.is_(True))
)).scalars().all()
assert set(remaining) == SYSTEM_NAMES
@pytest.mark.asyncio
async def test_reset_content_preserves_system_tags_and_applications(db):
"""Reset re-tags CONTENT concepts; hygiene flags describe the file itself
and survive with their applications."""
wip = await _system_tag(db, "wip")
plain = Tag(name="doomed concept", kind=TagKind.general)
db.add(plain)
img = ImageRecord(
path="/images/r.jpg", sha256="f" * 64, size_bytes=1, mime="image/jpeg",
width=1, height=1, origin="imported_filesystem",
integrity_status="unknown",
)
db.add(img)
await db.flush()
for tid in (wip.id, plain.id):
await db.execute(insert(image_tag).values(
image_record_id=img.id, tag_id=tid, source="manual",
))
await db.commit()
result = await db.run_sync(
lambda s: reset_content_tagging(s, dry_run=False)
)
assert result["deleted"] == 1 # only the plain general tag
names_left = (await db.execute(
select(Tag.name).where(Tag.kind == TagKind.general)
)).scalars().all()
assert "doomed concept" not in names_left
assert "wip" in names_left
wip_apps = (await db.execute(
select(image_tag.c.image_record_id)
.where(image_tag.c.tag_id == wip.id)
)).scalars().all()
assert wip_apps == [img.id]
@pytest.mark.asyncio
async def test_merge_into_system_tag_allowed(client, db):
wip = await _system_tag(db, "wip")
created = await client.post(
"/api/tags", json={"name": "old wip", "kind": "general"}
)
source_id = (await created.get_json())["id"]
resp = await client.post(
f"/api/tags/{source_id}/merge", json={"target_id": wip.id}
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["target"]["id"] == wip.id
assert body["source_deleted"] is True
+13 -4
View File
@@ -10,20 +10,29 @@ def test_serialize_tag_with_enum_kind():
# ORM/column rows carry kind as a TagKind enum.
row = SimpleNamespace(
id=1, name="Sasuke Uchiha", kind=TagKind.character,
fandom_id=5, fandom_name="Naruto",
fandom_id=5, fandom_name="Naruto", is_system=False,
)
assert serialize_tag(row) == {
"id": 1, "name": "Sasuke Uchiha", "kind": "character",
"fandom_id": 5, "fandom_name": "Naruto",
"fandom_id": 5, "fandom_name": "Naruto", "is_system": False,
}
def test_serialize_tag_with_string_kind_and_no_fandom():
# autocomplete hits already store kind as a plain string.
# autocomplete hits already store kind as a plain string. A row from a
# select that predates the is_system column serializes as non-system.
row = SimpleNamespace(
id=2, name="solo", kind="general", fandom_id=None, fandom_name=None,
)
assert serialize_tag(row) == {
"id": 2, "name": "solo", "kind": "general",
"fandom_id": None, "fandom_name": None,
"fandom_id": None, "fandom_name": None, "is_system": False,
}
def test_serialize_tag_system_row():
row = SimpleNamespace(
id=3, name="wip", kind=TagKind.general,
fandom_id=None, fandom_name=None, is_system=True,
)
assert serialize_tag(row)["is_system"] is True
+125
View File
@@ -0,0 +1,125 @@
"""Training hygiene (#128): system-tagged images are ABSENT from other
concepts' training data and from CCIP reference prototypes.
sklearn only exists in the ml image, so these pin head_training_ids (the
sklearn-free selection split out of train_head) rather than a full fit —
the exclusion lives entirely in that selection.
"""
import pytest
from sqlalchemy import insert, select
from backend.app.models import ImageRecord, ImageRegion, Tag, TagKind
from backend.app.models.tag import image_tag
from backend.app.services.ml import ccip
from backend.app.services.ml.heads import head_training_ids
from backend.app.services.ml.training_data import _hygiene_excluded_ids
from backend.app.services.tag_service import TagService
pytestmark = pytest.mark.integration
_CFG = {"min_positives": 2, "neg_ratio": 1}
async def _system_wip(db) -> Tag:
return (await db.execute(
select(Tag).where(Tag.is_system.is_(True), Tag.name == "wip")
)).scalar_one()
async def _img(db, sha, *, embedded=True):
rec = ImageRecord(
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
width=1, height=1, origin="imported_filesystem",
integrity_status="unknown",
siglip_embedding=([0.1] * 1152 if embedded else None),
)
db.add(rec)
await db.flush()
return rec
async def _apply(db, image_id, tag_id):
await db.execute(insert(image_tag).values(
image_record_id=image_id, tag_id=tag_id, source="manual",
))
@pytest.mark.asyncio
async def test_hygiene_excluded_ids(db):
wip = await _system_wip(db)
flagged = await _img(db, "a" * 64)
await _img(db, "b" * 64)
await _apply(db, flagged.id, wip.id)
excluded = await db.run_sync(lambda s: _hygiene_excluded_ids(s))
assert excluded == {flagged.id}
@pytest.mark.asyncio
async def test_head_selection_drops_hygiene_from_both_sides(db):
"""A wip-tagged image of concept X is neither a positive NOR a sampled
negative for X — it is absent entirely."""
wip = await _system_wip(db)
concept = await TagService(db).find_or_create("concept_x", TagKind.general)
clean_a = await _img(db, "a" * 64)
clean_b = await _img(db, "b" * 64)
flagged_pos = await _img(db, "c" * 64) # X + wip: dropped positive
negative_pool = await _img(db, "d" * 64) # untagged: legit negative
flagged_pool = await _img(db, "e" * 64) # wip only: must not be sampled
for img in (clean_a, clean_b, flagged_pos):
await _apply(db, img.id, concept.id)
await _apply(db, flagged_pos.id, wip.id)
await _apply(db, flagged_pool.id, wip.id)
ids = await db.run_sync(lambda s: head_training_ids(s, concept.id, _CFG))
assert ids is not None
pos_ids, neg_ids = ids
assert set(pos_ids) == {clean_a.id, clean_b.id}
assert negative_pool.id in neg_ids
assert flagged_pos.id not in neg_ids
assert flagged_pool.id not in neg_ids
@pytest.mark.asyncio
async def test_system_tags_own_head_keeps_hygiene_positives(db):
"""The wip head itself trains ON wip-tagged images — that's what makes
auto-flagging work."""
wip = await _system_wip(db)
one = await _img(db, "a" * 64)
two = await _img(db, "b" * 64)
await _img(db, "c" * 64) # negative pool
await _apply(db, one.id, wip.id)
await _apply(db, two.id, wip.id)
ids = await db.run_sync(lambda s: head_training_ids(s, wip.id, _CFG))
assert ids is not None
pos_ids, _ = ids
assert set(pos_ids) == {one.id, two.id}
@pytest.mark.asyncio
async def test_ccip_references_skip_hygiene_images(db):
"""A wip's figure region must never become an identity prototype, even on
a single-character image."""
ccip._REF_CACHE.update(sig=None, refs=None)
wip = await _system_wip(db)
char = await TagService(db).find_or_create("Char A", TagKind.character)
clean = await _img(db, "a" * 64)
flagged = await _img(db, "b" * 64)
for img, slot in ((clean, 0), (flagged, 1)):
vec = [0.0] * 768
vec[slot] = 1.0
db.add(ImageRegion(
image_record_id=img.id, kind="figure",
rx=0.0, ry=0.0, rw=1.0, rh=1.0,
ccip_embedding=vec, embedding_version="ccip-test",
))
await _apply(db, img.id, char.id)
await _apply(db, flagged.id, wip.id)
await db.commit()
refs = await ccip.character_references(db)
vectors = refs.get(char.id, [])
assert len(vectors) == 1
assert float(vectors[0][0]) == pytest.approx(1.0)
assert float(vectors[0][1]) == pytest.approx(0.0)