diff --git a/alembic/versions/0075_tag_is_system.py b/alembic/versions/0075_tag_is_system.py new file mode 100644 index 0000000..a6b7e7a --- /dev/null +++ b/alembic/versions/0075_tag_is_system.py @@ -0,0 +1,60 @@ +"""tag.is_system + seed the three hygiene system tags + +Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a +character poison that character's head and CCIP references; banners/editor +screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the +product ships — not operator configuration — so the seed lives here. + +Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive, +matching TagService.rename's collision stance) instead of inserting a +duplicate, so an operator who already tagged `wip` keeps their applications. + +Revision ID: 0075 +Revises: 0074 +Create Date: 2026-07-03 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0075" +down_revision: Union[str, None] = "0074" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") + + +def upgrade() -> None: + op.add_column( + "tag", + sa.Column( + "is_system", sa.Boolean(), nullable=False, + server_default=sa.false(), + ), + ) + conn = op.get_bind() + for name in SYSTEM_TAG_NAMES: + adopted = conn.execute( + sa.text( + "UPDATE tag SET is_system = true " + "WHERE lower(name) = lower(:name) AND kind = 'general'" + ), + {"name": name}, + ) + if adopted.rowcount == 0: + conn.execute( + sa.text( + "INSERT INTO tag (name, kind, is_system) " + "VALUES (:name, 'general', true)" + ), + {"name": name}, + ) + + +def downgrade() -> None: + # The seeded rows survive as ordinary general tags — dropping the flag is + # enough to disarm the mechanism, and deleting rows would orphan any + # operator applications made while the flag existed. + op.drop_column("tag", "is_system") diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index d94615b..e1ef468 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -156,6 +156,10 @@ async def tag_delete(tag_id: int): ) except LookupError: return _bad("not_found", status=404) + except ValueError as exc: + # System tags (#128) — the training-hygiene machinery keys on + # these rows. + return _bad("system_tag", detail=str(exc)) return jsonify(result) diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index ec9990a..ed156c8 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -324,6 +324,7 @@ async def get_tag(tag_id: int): "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id, + "is_system": tag.is_system, } ) @@ -390,6 +391,7 @@ async def update_tag(tag_id: int): "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id, + "is_system": tag.is_system, } ) diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index a74575b..2479578 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -10,6 +10,7 @@ from datetime import datetime from enum import StrEnum from sqlalchemy import ( + Boolean, CheckConstraint, Column, DateTime, @@ -17,6 +18,7 @@ from sqlalchemy import ( Integer, String, Table, + false, func, ) from sqlalchemy import ( @@ -41,6 +43,12 @@ class TagKind(StrEnum): # to keep historic tag rows queryable. +# The seeded system tags (migration 0075). PRESENTATION tags additionally +# hide from whole-image similarity results — they cluster on UI chrome, not +# content. `wip` is real art: only the training pipelines exclude it. +SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") +PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot") + image_tag = Table( "image_tag", Base.metadata, @@ -74,6 +82,14 @@ class Tag(Base): fandom_id: Mapped[int | None] = mapped_column( ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True ) + # System tags ship with FC (wip / banner / editor screenshot, seeded in + # migration 0075) and drive the training-hygiene exclusions: images + # carrying one are excluded from OTHER concepts' head training and from + # CCIP identity references. The mechanism keys on these exact rows, so + # they're protected from rename/merge-away/re-fandom in TagService. + is_system: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=false() + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 8708e28..cd7ef99 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -from sqlalchemy import delete, func, or_, select, update +from sqlalchemy import and_, delete, func, or_, select, update from sqlalchemy.orm import Session, aliased from ..models import ( @@ -203,6 +203,9 @@ def _unused_tag_conditions() -> list: Tag.id.not_in(used_via_series), Tag.id.not_in(used_via_chapter), Tag.id.not_in(used_via_fandom), + # System tags (#128) ship with zero applications and must survive a + # prune — the training-hygiene machinery keys on the rows. + Tag.is_system.is_(False), ] @@ -402,6 +405,8 @@ def delete_tag(session: Session, *, tag_id: int) -> dict: tag = session.get(Tag, tag_id) if tag is None: raise LookupError(f"tag id not found: {tag_id}") + if tag.is_system: + raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted") associations_count = count_tag_associations(session, tag_id=tag_id) info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value} session.delete(tag) @@ -731,7 +736,10 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict: heads retrain from whatever the operator re-tags. (The API route gates the live run behind a preview-derived confirm token for exactly this reason.) - PRESERVED: fandom + series tags and their series_page ordering. CASCADE on + PRESERVED: fandom + series tags and their series_page ordering, AND the + system hygiene tags (#128) WITH their applications — the reset re-tags + CONTENT concepts, while wip/banner flags describe the file itself and + re-flagging hundreds of banners by hand would be pure loss. CASCADE on image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting character tags never touches the fandom rows. Irreversible except via DB backup @@ -744,7 +752,9 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict: "sample_names": [first 50], and on live runs "deleted": total} """ - predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS) + predicate = and_( + Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False) + ) rows = session.execute( select(Tag.id, Tag.name, Tag.kind).where(predicate) ).all() diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index e385459..4cde351 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag -from ..models.tag import image_tag +from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag from .pagination import decode_cursor, encode_cursor from .tag_query import ( fandom_join_alias, @@ -693,9 +693,23 @@ class GalleryService: eff = _effective_date_col() stmt = select(ImageRecord, Post.post_date, eff.label("eff")) stmt = _outer_join_primary_post(stmt) + # Presentation images (banner / editor-screenshot system tags, #128) + # cluster on UI chrome rather than content, so near any one of them + # they'd fill the grid. Excluded from CANDIDATES only — the anchor + # itself may be a banner, and `wip` stays surfaced (real art; only + # the training pipelines exclude it). + presentation = ( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where( + Tag.is_system.is_(True), + Tag.name.in_(PRESENTATION_SYSTEM_TAGS), + ) + ) stmt = stmt.where( ImageRecord.siglip_embedding.is_not(None), ImageRecord.id != image_id, + ImageRecord.id.not_in(presentation), ) stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=None, diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 2912ccd..8c98edd 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -62,6 +62,18 @@ def _single_character_images(): ) +def _hygiene_tagged_images(): + """Subquery of image ids carrying any SYSTEM tag (wip / banner / editor + screenshot). Training hygiene (#128): such images never contribute + reference prototypes — a faceless wip's figure region would otherwise + become an identity reference for the character it's tagged with.""" + return ( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ) + + async def _ref_signature(session: AsyncSession) -> tuple: n_tags = ( await session.execute( @@ -79,7 +91,17 @@ async def _ref_signature(session: AsyncSession) -> tuple: ) ) ).one() - return (n_tags, n_regs, max_id) + # Hygiene applications must invalidate too: tagging an image `wip` changes + # the reference set without touching character-tag or region counts. + n_hygiene = ( + await session.execute( + select(func.count()) + .select_from(image_tag) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ) + ).scalar_one() + return (n_tags, n_regs, max_id, n_hygiene) async def character_references(session: AsyncSession) -> dict[int, list]: @@ -102,6 +124,9 @@ async def character_references(session: AsyncSession) -> dict[int, list]: .where(ImageRegion.kind.in_(_FIGURE_KINDS)) .where(ImageRegion.ccip_embedding.is_not(None)) .where(ImageRegion.image_record_id.in_(_single_character_images())) + .where( + ImageRegion.image_record_id.not_in(_hygiene_tagged_images()) + ) ) ).all() refs: dict[int, list] = {} diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index 2f47401..d7d8480 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -40,6 +40,7 @@ from ...models import ( from ...models.tag import image_tag from .training_data import ( _auto_apply_point, + _hygiene_excluded_ids, _ids_with_tag, _l2norm, _load_embeddings, @@ -150,11 +151,16 @@ def train_all_heads( embedding_version = _embedder_version(session) eligible = _eligible_tag_ids(session, cfg["min_positives"]) eligible_set = set(eligible) + # Computed once per run, not per head — the hygiene set is identical for + # every non-system concept. + hygiene = _hygiene_excluded_ids(session) trained = 0 skipped = 0 for i, tag_id in enumerate(eligible): try: - ok = train_head(session, tag_id, embedding_version, cfg, np) + ok = train_head( + session, tag_id, embedding_version, cfg, np, hygiene=hygiene + ) except Exception: log.exception("train_head failed for tag %d", tag_id) ok = False @@ -174,27 +180,56 @@ def train_all_heads( return {"n_trained": trained, "n_skipped": skipped} +def head_training_ids( + session: Session, tag_id: int, cfg: dict, hygiene: set[int] | None = None, +) -> tuple[list[int], list[int]] | None: + """Select (pos_ids, neg_ids) for one head. Split out of train_head and + kept sklearn-free so the hygiene exclusion is testable in the CI env + (sklearn only exists in the ml image). Returns None when the concept has + too few usable positives. + + Training hygiene (#128): images carrying a system tag are ABSENT from + every other concept's training — dropped as positives AND kept out of + the rejection/sampled negative pool (see _hygiene_excluded_ids). A system + tag's own head trains on them unfiltered: its positives ARE the hygiene + images.""" + tag = session.get(Tag, tag_id) + if tag is not None and tag.is_system: + hygiene = set() + elif hygiene is None: + hygiene = _hygiene_excluded_ids(session) + + pos_ids = [i for i in _ids_with_tag(session, tag_id) if i not in hygiene] + if len(pos_ids) < cfg["min_positives"]: + return None + + pos_set = set(pos_ids) + rejected = [ + i for i in _rejected_ids(session, tag_id) + if i not in pos_set and i not in hygiene + ] + want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4) + sampled = _sample_unlabeled( + session, pos_set | set(rejected) | hygiene, + min(_UNLABELED_POOL, want_neg), + ) + return pos_ids, rejected + [i for i in sampled if i not in pos_set] + + def train_head( - session: Session, tag_id: int, embedding_version: str, cfg: dict, np + session: Session, tag_id: int, embedding_version: str, cfg: dict, np, + hygiene: set[int] | None = None, ) -> bool: """Fit + upsert one head. Returns True if a head was written, False if the concept had too few usable examples to train (the row is then removed).""" from sklearn.linear_model import LogisticRegression from sklearn.model_selection import StratifiedKFold, cross_val_predict - pos_ids = _ids_with_tag(session, tag_id) - if len(pos_ids) < cfg["min_positives"]: + ids = head_training_ids(session, tag_id, cfg, hygiene) + if ids is None: session.execute(delete(TagHead).where(TagHead.tag_id == tag_id)) return False - - pos_set = set(pos_ids) - rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set] - want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4) - sampled = _sample_unlabeled( - session, pos_set | set(rejected), min(_UNLABELED_POOL, want_neg) - ) - neg_ids = rejected + [i for i in sampled if i not in pos_set] - + pos_ids, neg_ids = ids emb = _load_embeddings(session, pos_ids + neg_ids) pos = [emb[i] for i in pos_ids if i in emb] neg = [emb[i] for i in neg_ids if i in emb] diff --git a/backend/app/services/ml/training_data.py b/backend/app/services/ml/training_data.py index d01acf7..5e2963a 100644 --- a/backend/app/services/ml/training_data.py +++ b/backend/app/services/ml/training_data.py @@ -17,10 +17,33 @@ from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session -from ...models import ImageRecord, TagSuggestionRejection +from ...models import ImageRecord, Tag, TagSuggestionRejection from ...models.tag import image_tag +def _hygiene_excluded_ids(session: Session) -> set[int]: + """Ids of images carrying ANY system tag (wip / banner / editor + screenshot — milestone #128). These images are excluded from OTHER + concepts' head training entirely: not positives (a rough wip tagged as a + character drags that head toward 'generic sketch') and not sampled or + rejection negatives (a wip OF character X is not evidence against X) — + simply absent. A system tag's OWN head trains on them unchanged; that is + what makes auto-flagging banners/editor screenshots work. + + Item-level by design: a wip-tagged process video contributes (or + withholds) ALL its sampled frames, though some may show the finished + piece. Operator call 2026-07-03: with enough clean data this washes out — + no per-frame handling. + """ + return set( + session.execute( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ).scalars().all() + ) + + def _ids_with_tag(session: Session, tag_id: int) -> list[int]: return [ r[0] for r in session.execute( diff --git a/backend/app/services/platforms/subscribestar.py b/backend/app/services/platforms/subscribestar.py index 554c26c..886272b 100644 --- a/backend/app/services/platforms/subscribestar.py +++ b/backend/app/services/platforms/subscribestar.py @@ -14,9 +14,11 @@ `_personalization_id` age-confirmation cookie that the user can't easily refresh — SubscribeStar's frontend JS uses localStorage to suppress the age popup once dismissed. gallery-dl's own login flow - sidesteps this by setting `18_plus_agreement_generic=true` on - `.subscribestar.adult`; we mirror that for cookies captured via the - extension. + sidesteps this by setting `18_plus_agreement_generic=true`; we mirror + that for cookies captured via the extension — on EVERY SubscribeStar + domain, because cookies are domain-scoped and the wall exists on all + of them: an .adult-only line never rides along to a .art creator page + (Elasid, event #54116 — every poll 302'd to /age_confirmation_warning). """ from .base import GD_DEFAULTS, PlatformInfo, str_id_value @@ -29,20 +31,31 @@ def derive_post_url(data: dict) -> str | None: return None +# All domains SubscribeStar serves creators from; the age wall gates each. +_SS_DOMAINS = (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art") + + def augment_cookies(netscape: str) -> str: - if "18_plus_agreement_generic" in netscape: - return netscape # Far-future expiry — gallery-dl's own login flow sets this with no # explicit expiry; the server only checks presence/value. expiry = 4102444800 # 2100-01-01 UTC - line = "\t".join([ - ".subscribestar.adult", "TRUE", "/", "TRUE", - str(expiry), "18_plus_agreement_generic", "true", - ]) body = netscape.rstrip("\n") if not body: body = "# Netscape HTTP Cookie File" - return body + "\n" + line + "\n" + lines = netscape.splitlines() + for domain in _SS_DOMAINS: + # Per-domain presence check: captured cookies carrying the age + # cookie for one domain must not suppress injection on the others. + if any( + line.startswith(domain + "\t") and "18_plus_agreement_generic" in line + for line in lines + ): + continue + body += "\n" + "\t".join([ + domain, "TRUE", "/", "TRUE", + str(expiry), "18_plus_agreement_generic", "true", + ]) + return body + "\n" INFO = PlatformInfo( @@ -51,10 +64,11 @@ INFO = PlatformInfo( description="Download posts from SubscribeStar creators", auth_type="cookies", requires_auth=True, - url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", + url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult|art)/", url_examples=[ "https://subscribestar.adult/example_artist", "https://www.subscribestar.com/example_artist", + "https://subscribestar.art/example_artist", ], default_config={**GD_DEFAULTS, "content_types": ["all"]}, derive_post_url=derive_post_url, diff --git a/backend/app/services/tag_directory_service.py b/backend/app/services/tag_directory_service.py index 49c55cc..a6b3756 100644 --- a/backend/app/services/tag_directory_service.py +++ b/backend/app/services/tag_directory_service.py @@ -107,6 +107,7 @@ class TagDirectoryService: "kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind, "fandom_id": tag.fandom_id, "fandom_name": fandom_name, + "is_system": tag.is_system, "image_count": int(image_count), "preview_thumbnails": previews.get(tag.id, []), } diff --git a/backend/app/services/tag_query.py b/backend/app/services/tag_query.py index 1b7db59..13186dc 100644 --- a/backend/app/services/tag_query.py +++ b/backend/app/services/tag_query.py @@ -67,24 +67,28 @@ def fandom_join_alias(): def tag_columns(fandom_alias): - """The canonical (id, name, kind, fandom_id, fandom_name) column set for a - tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`).""" + """The canonical (id, name, kind, fandom_id, fandom_name, is_system) + column set for a tag select that outerjoins `fandom_alias` (from + `fandom_join_alias()`).""" return [ Tag.id, Tag.name, Tag.kind, Tag.fandom_id, fandom_alias.c.name.label("fandom_name"), + Tag.is_system, ] def serialize_tag(row) -> dict: """Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the - canonical tag dict. `kind` may be a TagKind enum or a plain string.""" + canonical tag dict. `kind` may be a TagKind enum or a plain string. + `is_system` defaults False for callers whose selects predate the column.""" return { "id": row.id, "name": row.name, "kind": row.kind.value if hasattr(row.kind, "value") else row.kind, "fandom_id": row.fandom_id, "fandom_name": row.fandom_name, + "is_system": bool(getattr(row, "is_system", False)), } diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index a3ca07e..98e1d9c 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -330,6 +330,12 @@ class TagService: tag = await self.session.get(Tag, tag_id) if tag is None: raise TagValidationError(f"Tag {tag_id} not found") + # The training-hygiene machinery keys on system tags by row; a rename + # would silently disarm it (milestone #128). + if tag.is_system: + raise TagValidationError( + f"'{tag.name}' is a system tag and cannot be renamed" + ) # Case-insensitive clash (#701) — renaming onto a differently-cased tag # is still a merge, not a silent fork. @@ -528,6 +534,14 @@ class TagService: source = await self.session.get(Tag, source_id) if source is None: raise TagValidationError(f"Tag {source_id} not found") + # Merge deletes the source row — the only tag-delete path — and the + # training-hygiene machinery keys on system rows (milestone #128). + # Merging INTO a system tag stays allowed (adopting an operator's old + # 'wip sketch' tag into the system 'wip' is the intended move). + if source.is_system: + raise TagValidationError( + f"'{source.name}' is a system tag and cannot be merged away" + ) target = await self.session.get(Tag, target_id) if target is None: raise TagValidationError(f"Tag {target_id} not found") @@ -792,6 +806,10 @@ async def normalize_existing_tags( rows = ( await session.execute( select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id) + # System tags (#128) are exempt: their names are the keys the + # hygiene machinery matches on, and canonicalization would recase + # 'wip' → 'Wip' (this path bypasses TagService.rename's guard). + .where(Tag.is_system.is_(False)) ) ).all() groups = _group_existing_tags(rows) diff --git a/frontend/src/components/discovery/TagCard.vue b/frontend/src/components/discovery/TagCard.vue index 1d1a0b1..9aa7582 100644 --- a/frontend/src/components/discovery/TagCard.vue +++ b/frontend/src/components/discovery/TagCard.vue @@ -13,9 +13,15 @@