From b54243a1ff56aedd5b0ca71ada47a822676d0b56 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 21:51:51 -0400 Subject: [PATCH 1/9] fix(subscribestar): inject the 18+ age cookie on every SubscribeStar domain The cookie was pinned to .subscribestar.adult only; cookies are domain-scoped, so sources on subscribestar.art (Elasid, event #54116) never sent it and every poll 302d to /age_confirmation_warning. Emit one line per domain (.com/.adult/.art) with a per-domain presence check, and admit .art in the platform url_pattern. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- .../app/services/platforms/subscribestar.py | 36 +++++++++++++------ tests/test_credential_service.py | 28 ++++++++++----- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/backend/app/services/platforms/subscribestar.py b/backend/app/services/platforms/subscribestar.py index 554c26c..886272b 100644 --- a/backend/app/services/platforms/subscribestar.py +++ b/backend/app/services/platforms/subscribestar.py @@ -14,9 +14,11 @@ `_personalization_id` age-confirmation cookie that the user can't easily refresh — SubscribeStar's frontend JS uses localStorage to suppress the age popup once dismissed. gallery-dl's own login flow - sidesteps this by setting `18_plus_agreement_generic=true` on - `.subscribestar.adult`; we mirror that for cookies captured via the - extension. + sidesteps this by setting `18_plus_agreement_generic=true`; we mirror + that for cookies captured via the extension — on EVERY SubscribeStar + domain, because cookies are domain-scoped and the wall exists on all + of them: an .adult-only line never rides along to a .art creator page + (Elasid, event #54116 — every poll 302'd to /age_confirmation_warning). """ from .base import GD_DEFAULTS, PlatformInfo, str_id_value @@ -29,20 +31,31 @@ def derive_post_url(data: dict) -> str | None: return None +# All domains SubscribeStar serves creators from; the age wall gates each. +_SS_DOMAINS = (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art") + + def augment_cookies(netscape: str) -> str: - if "18_plus_agreement_generic" in netscape: - return netscape # Far-future expiry — gallery-dl's own login flow sets this with no # explicit expiry; the server only checks presence/value. expiry = 4102444800 # 2100-01-01 UTC - line = "\t".join([ - ".subscribestar.adult", "TRUE", "/", "TRUE", - str(expiry), "18_plus_agreement_generic", "true", - ]) body = netscape.rstrip("\n") if not body: body = "# Netscape HTTP Cookie File" - return body + "\n" + line + "\n" + lines = netscape.splitlines() + for domain in _SS_DOMAINS: + # Per-domain presence check: captured cookies carrying the age + # cookie for one domain must not suppress injection on the others. + if any( + line.startswith(domain + "\t") and "18_plus_agreement_generic" in line + for line in lines + ): + continue + body += "\n" + "\t".join([ + domain, "TRUE", "/", "TRUE", + str(expiry), "18_plus_agreement_generic", "true", + ]) + return body + "\n" INFO = PlatformInfo( @@ -51,10 +64,11 @@ INFO = PlatformInfo( description="Download posts from SubscribeStar creators", auth_type="cookies", requires_auth=True, - url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", + url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult|art)/", url_examples=[ "https://subscribestar.adult/example_artist", "https://www.subscribestar.com/example_artist", + "https://subscribestar.art/example_artist", ], default_config={**GD_DEFAULTS, "content_types": ["all"]}, derive_post_url=derive_post_url, diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py index b9857ec..e317f52 100644 --- a/tests/test_credential_service.py +++ b/tests/test_credential_service.py @@ -116,8 +116,10 @@ async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp cookie; the browser-stored cookie expires annually and can't be easily refreshed (the JS age popup is suppressed by localStorage). Mirror gallery-dl's own login-flow workaround by injecting - `18_plus_agreement_generic=true` on `.subscribestar.adult` whenever - cookies for subscribestar are materialized.""" + `18_plus_agreement_generic=true` whenever cookies for subscribestar + are materialized — on EVERY SubscribeStar domain, since cookies are + domain-scoped and creators live on .com/.adult/.art alike (a .adult-only + line left .art sources stuck on the age wall — Elasid, event #54116).""" netscape_in = ( "# Netscape HTTP Cookie File\n" ".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n" @@ -126,16 +128,18 @@ async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in) path = await svc.get_cookies_path("subscribestar") contents = path.read_text() - assert "18_plus_agreement_generic\ttrue" in contents - assert ".subscribestar.adult" in contents + for domain in (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art"): + assert f"{domain}\tTRUE\t/\tTRUE\t4102444800\t18_plus_agreement_generic\ttrue" in contents # Original session_id cookie preserved. assert "session_id\txyz" in contents @pytest.mark.asyncio -async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto, tmp_path): - """If the operator's captured cookies ALREADY contain the age cookie - (e.g. a manual paste, or a re-login), don't double-inject.""" +async def test_get_cookies_path_subscribestar_idempotent_per_domain(db, crypto, tmp_path): + """If the operator's captured cookies ALREADY contain the age cookie for + a domain (e.g. a manual paste, or a re-login), don't double-inject on + THAT domain — but a .adult-only capture must still get the cookie added + for .com and .art (per-domain check, not presence-anywhere).""" netscape_in = ( "# Netscape HTTP Cookie File\n" ".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\t18_plus_agreement_generic\ttrue\n" @@ -145,7 +149,15 @@ async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in) path = await svc.get_cookies_path("subscribestar") contents = path.read_text() - assert contents.count("18_plus_agreement_generic") == 1 + assert contents.count("18_plus_agreement_generic") == 3 + adult_lines = [ + line for line in contents.splitlines() + if line.startswith(".subscribestar.adult\t") + and "18_plus_agreement_generic" in line + ] + assert len(adult_lines) == 1 + assert ".subscribestar.com\t" in contents + assert ".subscribestar.art\t" in contents @pytest.mark.asyncio From 6d314d662fcaa6c4552d56304c32f8b7e5a8d2aa Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 22:00:40 -0400 Subject: [PATCH 2/9] feat(modal): applied tags drop from search instantly; manual add == accepting a suggestion Three tag-flow gaps in the view modal (and the Explore workspace, which shares TagPanel): - the type-to-add dropdown now filters both its sections against the imageledger applied tags reactively, so a just-added tag disappears from search the moment the chip rail updates instead of after a modal refresh - manually picking or creating a tag the model also suggested routes through the suggestion-accept flow: the acceptance is recorded for head training and the row leaves the panel, instead of the add silently bypassing the feedback loop - removing a tag reloads the suggestion lists, so a model-suggested tag returns to the suggestions area (flagged rejected, one-click reversible) rather than vanishing until the next modal open Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- .../src/components/modal/TagAutocomplete.vue | 28 +++++- frontend/src/components/modal/TagPanel.vue | 49 +++++++++- frontend/src/stores/explore.js | 3 + frontend/src/stores/modal.js | 3 + frontend/src/stores/suggestions.js | 26 +++++- frontend/test/suggestions.spec.js | 90 +++++++++++++++++++ 6 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 frontend/test/suggestions.spec.js diff --git a/frontend/src/components/modal/TagAutocomplete.vue b/frontend/src/components/modal/TagAutocomplete.vue index 65f3527..bea6bcc 100644 --- a/frontend/src/components/modal/TagAutocomplete.vue +++ b/frontend/src/components/modal/TagAutocomplete.vue @@ -100,6 +100,13 @@ import { useSuggestionsStore } from '../../stores/suggestions.js' import FandomPicker from './FandomPicker.vue' const emit = defineEmits(['pick-existing', 'pick-new', 'accept-suggestion', 'cancel']) +// The host surface's applied tags (host.current?.tags). Both dropdown sections +// filter against this so a just-added tag drops out of the search the moment +// the chip rail updates, not after the next modal refresh (operator-asked +// 2026-07-03). +const props = defineProps({ + appliedTags: { type: Array, default: () => [] }, +}) const store = useTagStore() // The image's ML (Camie) suggestions, surfaced inline in this dropdown so the // operator can pick a suggestion while typing instead of hunting for it in the @@ -208,6 +215,18 @@ const createLabel = computed(() => function scorePct (s) { return `${Math.round(s.score * 100)}%` } +const appliedIds = computed(() => new Set(props.appliedTags.map(t => t.id))) +const appliedNames = computed(() => + new Set(props.appliedTags.map(t => `${t.kind}:${(t.name || '').toLowerCase()}`)), +) + +// Server matches minus the image's applied tags. allowCreate/sameNameCharExists +// keep reading the RAW hits — they reason about the tag universe (does this tag +// exist?), not about this image. +const visibleHits = computed(() => + hits.value.filter(h => !appliedIds.value.has(h.id)), +) + // This image's suggestions that match the typed query, minus any the server // autocomplete already returned (same name+kind) so a tag never shows twice. // Sources the FULL prediction set (allByCategory, down to the store floor) — NOT @@ -218,7 +237,7 @@ function scorePct (s) { return `${Math.round(s.score * 100)}%` } const suggestionHits = computed(() => { const q = parsedName.value.toLowerCase() if (!q) return [] - const seen = new Set(hits.value.map(h => `${h.kind}:${h.name.toLowerCase()}`)) + const seen = new Set(visibleHits.value.map(h => `${h.kind}:${h.name.toLowerCase()}`)) const out = [] for (const list of Object.values(suggestions.allByCategory)) { for (const s of list || []) { @@ -226,7 +245,12 @@ const suggestionHits = computed(() => { // can show + un-reject them; keep them OUT of the type-to-add dropdown, // whose job is finding a tag to ADD (un-reject lives in the panel). if (s.rejected) continue + // Already on the image (matched by canonical id or name+kind): nothing + // to add. Covers the window between an add and the next suggestions + // fetch, where the stale row would otherwise still be pickable. + if (s.canonical_tag_id != null && appliedIds.value.has(s.canonical_tag_id)) continue const key = `${s.category}:${s.display_name.toLowerCase()}` + if (appliedNames.value.has(key)) continue if (!s.display_name.toLowerCase().includes(q)) continue if (seen.has(key)) continue seen.add(key) @@ -242,7 +266,7 @@ const suggestionHits = computed(() => { // One ordered list backing both the rendered dropdown and keyboard nav, so the // highlight index maps 1:1 to a row regardless of which section it's in. const rows = computed(() => { - const r = hits.value.map(h => ({ type: 'hit', key: `h${h.id}`, hit: h })) + const r = visibleHits.value.map(h => ({ type: 'hit', key: `h${h.id}`, hit: h })) for (const s of suggestionHits.value) { r.push({ type: 'suggestion', key: `s${s.category}:${s.display_name}`, sugg: s }) } diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue index a0619f6..d5e5639 100644 --- a/frontend/src/components/modal/TagPanel.vue +++ b/frontend/src/components/modal/TagPanel.vue @@ -15,6 +15,7 @@ @@ -107,17 +108,59 @@ defineExpose({ focusTagInput }) // (operator-asked 2026-06-26; the Explore workspace leans on this hard). async function onRemove(tagId) { errorMsg.value = null - try { await host.removeTag(tagId); focusTagInput() } + try { + await host.removeTag(tagId) + // removeTag also records a per-image dismissal server-side, so reloading + // puts a model-suggested tag BACK in the suggestions area (flagged + // rejected — visible + one-click reversible) instead of it vanishing + // until the next modal open (operator-asked 2026-07-03). + if (host.currentImageId != null) { + suggestions.load(host.currentImageId) + suggestions.loadAll(host.currentImageId) + } + focusTagInput() + } catch (e) { errorMsg.value = e.message } } async function onPickExisting(hit) { errorMsg.value = null - try { await host.addExistingTag(hit.id); focusTagInput() } + try { + // Manually picking a tag the model also suggested = accepting the + // suggestion: the acceptance is recorded and the row drops from the + // panel, instead of silently bypassing the feedback loop + // (operator-asked 2026-07-03). + const pending = suggestions.findPending(hit.kind, hit.name, hit.id) + if (pending) { + await suggestions.accept({ + ...pending, + canonical_tag_id: pending.canonical_tag_id ?? hit.id, + }) + await host.reloadTags() + } else { + await host.addExistingTag(hit.id) + } + focusTagInput() + } catch (e) { errorMsg.value = e.message } } async function onPickNew(payload) { errorMsg.value = null - try { await host.createAndAdd(payload); focusTagInput() } + try { + const imageId = host.currentImageId + const created = await host.createAndAdd(payload) + // The typed name can match a raw (or canonical) suggestion; creating it + // by hand still counts as accepting — record it and drop the row. Runs + // AFTER createAndAdd so a character's fandom pick is preserved (accept's + // own create path knows no fandom); the accept apply is idempotent. + // Guard on the image not having changed mid-create (fast prev/next). + if (created && host.currentImageId === imageId) { + const pending = suggestions.findPending(payload.kind, payload.name, created.id) + if (pending) { + await suggestions.accept({ ...pending, canonical_tag_id: created.id }) + } + } + focusTagInput() + } catch (e) { errorMsg.value = e.message } } // A suggestion picked from the autocomplete dropdown runs the SAME path as the diff --git a/frontend/src/stores/explore.js b/frontend/src/stores/explore.js index 8b5c45f..36ee046 100644 --- a/frontend/src/stores/explore.js +++ b/frontend/src/stores/explore.js @@ -176,6 +176,9 @@ export const useExploreStore = defineStore('explore', () => { } if (anchor.value?.id !== id) return // walked away mid-create await addExistingTag(tag.id) + // Returned so TagPanel can match the created tag against a pending + // suggestion and record the acceptance (manual add == accept, 2026-07-03). + return tag } // No overlay to dismiss in the Explore workspace — the chip-body "show me diff --git a/frontend/src/stores/modal.js b/frontend/src/stores/modal.js index 3859b61..567f867 100644 --- a/frontend/src/stores/modal.js +++ b/frontend/src/stores/modal.js @@ -170,6 +170,9 @@ export const useModalStore = defineStore('modal', () => { } if (currentImageId.value !== imageId) return // navigated away await addExistingTag(tag.id) + // Returned so TagPanel can match the created tag against a pending + // suggestion and record the acceptance (manual add == accept, 2026-07-03). + return tag } const isOpen = computed(() => currentImageId.value !== null) diff --git a/frontend/src/stores/suggestions.js b/frontend/src/stores/suggestions.js index 6a1198d..6271825 100644 --- a/frontend/src/stores/suggestions.js +++ b/frontend/src/stores/suggestions.js @@ -169,6 +169,29 @@ export const useSuggestionsStore = defineStore('suggestions', () => { toast({ text: `Alias removed: ${suggestion.display_name}`, type: 'success' }) } + // Find a live (non-rejected) pending suggestion matching a manually-picked + // tag — by canonical id when known, else by (kind, name). Manually adding a + // tag the model also suggested must be indistinguishable from accepting the + // suggestion (recorded + dropped from the panel), so TagPanel routes matches + // through accept() (operator-asked 2026-07-03). Searches the full dropdown + // set first, then the panel list (loadAll is best-effort and can be empty). + function findPending(kind, name, tagId = null) { + const lname = (name || '').toLowerCase() + for (const map of [allByCategory.value, byCategory.value]) { + for (const list of Object.values(map)) { + for (const s of list || []) { + if (s.rejected) continue + if (tagId != null && s.canonical_tag_id === tagId) return s + if ( + s.category === kind && + (s.display_name || '').toLowerCase() === lname + ) return s + } + } + } + return null + } + async function dismiss(suggestion) { const imageId = currentImageId if (imageId == null) return @@ -202,6 +225,7 @@ export const useSuggestionsStore = defineStore('suggestions', () => { return { byCategory, allByCategory, loading, error, - load, loadAll, accept, aliasAccept, removeAlias, dismiss, undismiss + load, loadAll, accept, aliasAccept, removeAlias, dismiss, undismiss, + findPending } }) diff --git a/frontend/test/suggestions.spec.js b/frontend/test/suggestions.spec.js new file mode 100644 index 0000000..0473db9 --- /dev/null +++ b/frontend/test/suggestions.spec.js @@ -0,0 +1,90 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useSuggestionsStore } from '../src/stores/suggestions.js' + +vi.mock('../src/utils/toast.js', () => ({ toast: vi.fn() })) + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, + statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)), + } + }) +} + +const sugg = (over = {}) => ({ + canonical_tag_id: 7, + display_name: 'Ichigo', + category: 'character', + score: 0.9, + source: 'head', + creates_new_tag: false, + rejected: false, + ...over, +}) + +// Seed the store through its own load/loadAll so currentImageId is set the +// way the panel sets it (accept() derives the image from it). +async function seed(store, byCategory) { + stubFetch(() => ({ status: 200, body: { by_category: byCategory } })) + await store.load(1) + await store.loadAll(1) +} + +describe('suggestions store: findPending (manual add == accept, 2026-07-03)', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('matches by canonical tag id', async () => { + const s = useSuggestionsStore() + await seed(s, { character: [sugg()] }) + expect(s.findPending('general', 'unrelated', 7)?.display_name).toBe('Ichigo') + }) + + it('matches raw suggestions by kind + name, case-insensitively', async () => { + const s = useSuggestionsStore() + await seed(s, { + general: [sugg({ + canonical_tag_id: null, display_name: 'Holding Sword', + category: 'general', creates_new_tag: true, + })], + }) + expect(s.findPending('general', 'holding sword')).toBeTruthy() + // Kind must match: same name under another kind is a different tag. + expect(s.findPending('character', 'holding sword')).toBeNull() + }) + + it('skips rejected rows and returns null when nothing matches', async () => { + const s = useSuggestionsStore() + await seed(s, { character: [sugg({ rejected: true })] }) + expect(s.findPending('character', 'Ichigo', 7)).toBeNull() + expect(s.findPending('character', 'Rukia')).toBeNull() + }) +}) + +describe('suggestions store: accept with a known tag id', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('POSTs only the accept endpoint (no tag create) and drops the row', async () => { + const s = useSuggestionsStore() + await seed(s, { character: [sugg()] }) + const calls = [] + stubFetch((url, init) => { + calls.push({ url, method: init?.method }) + return { status: 200, body: { accepted: true, tag_id: 7 } } + }) + // TagPanel routes a manual pick that matches a pending suggestion through + // accept, filling canonical_tag_id from the picked hit when the + // suggestion is raw — no /api/tags create must fire in that case. + await s.accept({ ...s.byCategory.character[0], canonical_tag_id: 7 }) + expect(calls).toHaveLength(1) + expect(calls[0].url).toContain('/api/images/1/suggestions/accept') + expect(s.byCategory.character).toHaveLength(0) + expect(s.allByCategory.character).toHaveLength(0) + }) +}) From 581b7785280682ac9562a355400dbd3335516851 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 22:02:24 -0400 Subject: [PATCH 3/9] fix(modal): accept-by-known-id keeps the raw suggestion row identity for the drop Spreading canonical_tag_id onto a raw suggestion changed its _keyOf identity, so _dropEverywhere missed the actual list row and the panel kept showing an already-accepted suggestion. Pass the resolved id as an option instead; pinned with a raw-suggestion spec. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- frontend/src/components/modal/TagPanel.vue | 7 ++--- frontend/src/stores/suggestions.js | 9 +++++-- frontend/test/suggestions.spec.js | 30 +++++++++++++++++++--- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue index d5e5639..39487c5 100644 --- a/frontend/src/components/modal/TagPanel.vue +++ b/frontend/src/components/modal/TagPanel.vue @@ -131,10 +131,7 @@ async function onPickExisting(hit) { // (operator-asked 2026-07-03). const pending = suggestions.findPending(hit.kind, hit.name, hit.id) if (pending) { - await suggestions.accept({ - ...pending, - canonical_tag_id: pending.canonical_tag_id ?? hit.id, - }) + await suggestions.accept(pending, { tagId: hit.id }) await host.reloadTags() } else { await host.addExistingTag(hit.id) @@ -156,7 +153,7 @@ async function onPickNew(payload) { if (created && host.currentImageId === imageId) { const pending = suggestions.findPending(payload.kind, payload.name, created.id) if (pending) { - await suggestions.accept({ ...pending, canonical_tag_id: created.id }) + await suggestions.accept(pending, { tagId: created.id }) } } focusTagInput() diff --git a/frontend/src/stores/suggestions.js b/frontend/src/stores/suggestions.js index 6271825..dfa71e0 100644 --- a/frontend/src/stores/suggestions.js +++ b/frontend/src/stores/suggestions.js @@ -97,7 +97,12 @@ export const useSuggestionsStore = defineStore('suggestions', () => { } } - async function accept(suggestion) { + // opts.tagId: the tag's known id, when the caller already created/resolved + // it (TagPanel's manual-add-matches-suggestion path). Passed separately — + // NOT spread onto the suggestion — because _dropEverywhere keys off the + // suggestion object, and mutating canonical_tag_id on a raw suggestion + // would stop it matching its own row in the lists. + async function accept(suggestion, { tagId: knownTagId = null } = {}) { // Capture imageId so a mid-flight prev/next can't reroute the // accept POST to a different image AND push the tag to that // image's allowlist. @@ -106,7 +111,7 @@ export const useSuggestionsStore = defineStore('suggestions', () => { // Raw tags (creates_new_tag) have no canonical_tag_id; the backend's // accept endpoint needs a tag_id, so for raw tags we create the tag // first via the existing /api/tags endpoint, then accept by id. - let tagId = suggestion.canonical_tag_id + let tagId = knownTagId ?? suggestion.canonical_tag_id if (tagId == null) { const created = await api.post('/api/tags', { body: { name: suggestion.display_name, kind: suggestion.category } diff --git a/frontend/test/suggestions.spec.js b/frontend/test/suggestions.spec.js index 0473db9..929e13e 100644 --- a/frontend/test/suggestions.spec.js +++ b/frontend/test/suggestions.spec.js @@ -78,13 +78,35 @@ describe('suggestions store: accept with a known tag id', () => { calls.push({ url, method: init?.method }) return { status: 200, body: { accepted: true, tag_id: 7 } } }) - // TagPanel routes a manual pick that matches a pending suggestion through - // accept, filling canonical_tag_id from the picked hit when the - // suggestion is raw — no /api/tags create must fire in that case. - await s.accept({ ...s.byCategory.character[0], canonical_tag_id: 7 }) + await s.accept(s.byCategory.character[0], { tagId: 7 }) expect(calls).toHaveLength(1) expect(calls[0].url).toContain('/api/images/1/suggestions/accept') expect(s.byCategory.character).toHaveLength(0) expect(s.allByCategory.character).toHaveLength(0) }) + + it('a RAW suggestion accepted with a known tagId skips create AND still drops its row', async () => { + // TagPanel's manual-create path: the tag was just created by + // host.createAndAdd, so accept must not create again — and the drop must + // key off the original raw suggestion object, not the resolved id + // (regression: spreading canonical_tag_id onto the suggestion changed its + // identity key and left the panel row behind). + const s = useSuggestionsStore() + const raw = sugg({ + canonical_tag_id: null, display_name: 'Holding Sword', + category: 'general', creates_new_tag: true, + }) + await seed(s, { general: [raw] }) + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: init?.body }) + return { status: 200, body: { accepted: true, tag_id: 42 } } + }) + await s.accept(s.byCategory.general[0], { tagId: 42 }) + expect(calls).toHaveLength(1) + expect(calls[0].url).toContain('/api/images/1/suggestions/accept') + expect(JSON.parse(calls[0].body)).toEqual({ tag_id: 42 }) + expect(s.byCategory.general).toHaveLength(0) + expect(s.allByCategory.general).toHaveLength(0) + }) }) From e9891ee9f34d490164eaa5a54430b073328bbede Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 23:14:49 -0400 Subject: [PATCH 4/9] =?UTF-8?q?feat(tags):=20system=20tags=20=E2=80=94=20i?= =?UTF-8?q?s=5Fsystem=20column,=20seeded=20hygiene=20tags,=20protection=20?= =?UTF-8?q?guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Training hygiene step 1 (milestone #128). Migration 0075 adds tag.is_system and seeds wip / banner / editor screenshot (kind=general), ADOPTING an existing same-(name,kind) tag case-insensitively instead of duplicating. These rows drive the upcoming training exclusions, so they are protected: rename and merge-away refuse system tags (merge-INTO stays allowed — folding an operator's old hygiene tag into the system row is the intended move; merge is the only tag-delete path, so that guard covers deletion). is_system rides every tag serialization. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- alembic/versions/0075_tag_is_system.py | 60 +++++++++++++++++++ backend/app/api/tags.py | 2 + backend/app/models/tag.py | 10 ++++ backend/app/services/tag_query.py | 10 +++- backend/app/services/tag_service.py | 14 +++++ tests/test_system_tags.py | 81 ++++++++++++++++++++++++++ tests/test_tag_query.py | 17 ++++-- 7 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 alembic/versions/0075_tag_is_system.py create mode 100644 tests/test_system_tags.py diff --git a/alembic/versions/0075_tag_is_system.py b/alembic/versions/0075_tag_is_system.py new file mode 100644 index 0000000..a6b7e7a --- /dev/null +++ b/alembic/versions/0075_tag_is_system.py @@ -0,0 +1,60 @@ +"""tag.is_system + seed the three hygiene system tags + +Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a +character poison that character's head and CCIP references; banners/editor +screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the +product ships — not operator configuration — so the seed lives here. + +Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive, +matching TagService.rename's collision stance) instead of inserting a +duplicate, so an operator who already tagged `wip` keeps their applications. + +Revision ID: 0075 +Revises: 0074 +Create Date: 2026-07-03 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0075" +down_revision: Union[str, None] = "0074" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") + + +def upgrade() -> None: + op.add_column( + "tag", + sa.Column( + "is_system", sa.Boolean(), nullable=False, + server_default=sa.false(), + ), + ) + conn = op.get_bind() + for name in SYSTEM_TAG_NAMES: + adopted = conn.execute( + sa.text( + "UPDATE tag SET is_system = true " + "WHERE lower(name) = lower(:name) AND kind = 'general'" + ), + {"name": name}, + ) + if adopted.rowcount == 0: + conn.execute( + sa.text( + "INSERT INTO tag (name, kind, is_system) " + "VALUES (:name, 'general', true)" + ), + {"name": name}, + ) + + +def downgrade() -> None: + # The seeded rows survive as ordinary general tags — dropping the flag is + # enough to disarm the mechanism, and deleting rows would orphan any + # operator applications made while the flag existed. + op.drop_column("tag", "is_system") diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index ec9990a..ed156c8 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -324,6 +324,7 @@ async def get_tag(tag_id: int): "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id, + "is_system": tag.is_system, } ) @@ -390,6 +391,7 @@ async def update_tag(tag_id: int): "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id, + "is_system": tag.is_system, } ) diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index a74575b..fe02dcd 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -10,6 +10,7 @@ from datetime import datetime from enum import StrEnum from sqlalchemy import ( + Boolean, CheckConstraint, Column, DateTime, @@ -17,6 +18,7 @@ from sqlalchemy import ( Integer, String, Table, + false, func, ) from sqlalchemy import ( @@ -74,6 +76,14 @@ class Tag(Base): fandom_id: Mapped[int | None] = mapped_column( ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True ) + # System tags ship with FC (wip / banner / editor screenshot, seeded in + # migration 0075) and drive the training-hygiene exclusions: images + # carrying one are excluded from OTHER concepts' head training and from + # CCIP identity references. The mechanism keys on these exact rows, so + # they're protected from rename/merge-away/re-fandom in TagService. + is_system: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=false() + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/services/tag_query.py b/backend/app/services/tag_query.py index 1b7db59..13186dc 100644 --- a/backend/app/services/tag_query.py +++ b/backend/app/services/tag_query.py @@ -67,24 +67,28 @@ def fandom_join_alias(): def tag_columns(fandom_alias): - """The canonical (id, name, kind, fandom_id, fandom_name) column set for a - tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`).""" + """The canonical (id, name, kind, fandom_id, fandom_name, is_system) + column set for a tag select that outerjoins `fandom_alias` (from + `fandom_join_alias()`).""" return [ Tag.id, Tag.name, Tag.kind, Tag.fandom_id, fandom_alias.c.name.label("fandom_name"), + Tag.is_system, ] def serialize_tag(row) -> dict: """Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the - canonical tag dict. `kind` may be a TagKind enum or a plain string.""" + canonical tag dict. `kind` may be a TagKind enum or a plain string. + `is_system` defaults False for callers whose selects predate the column.""" return { "id": row.id, "name": row.name, "kind": row.kind.value if hasattr(row.kind, "value") else row.kind, "fandom_id": row.fandom_id, "fandom_name": row.fandom_name, + "is_system": bool(getattr(row, "is_system", False)), } diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index a3ca07e..3bda769 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -330,6 +330,12 @@ class TagService: tag = await self.session.get(Tag, tag_id) if tag is None: raise TagValidationError(f"Tag {tag_id} not found") + # The training-hygiene machinery keys on system tags by row; a rename + # would silently disarm it (milestone #128). + if tag.is_system: + raise TagValidationError( + f"'{tag.name}' is a system tag and cannot be renamed" + ) # Case-insensitive clash (#701) — renaming onto a differently-cased tag # is still a merge, not a silent fork. @@ -528,6 +534,14 @@ class TagService: source = await self.session.get(Tag, source_id) if source is None: raise TagValidationError(f"Tag {source_id} not found") + # Merge deletes the source row — the only tag-delete path — and the + # training-hygiene machinery keys on system rows (milestone #128). + # Merging INTO a system tag stays allowed (adopting an operator's old + # 'wip sketch' tag into the system 'wip' is the intended move). + if source.is_system: + raise TagValidationError( + f"'{source.name}' is a system tag and cannot be merged away" + ) target = await self.session.get(Tag, target_id) if target is None: raise TagValidationError(f"Tag {target_id} not found") diff --git a/tests/test_system_tags.py b/tests/test_system_tags.py new file mode 100644 index 0000000..59a7763 --- /dev/null +++ b/tests/test_system_tags.py @@ -0,0 +1,81 @@ +"""System tags (milestone #128): seeded rows, protection guards, serialization. + +The three hygiene tags (wip / banner / editor screenshot) ship via migration +0075 and drive the training-hygiene exclusions. They must exist after +migration, serialize with is_system, and refuse rename/merge-away — the +mechanism keys on these exact rows. Merge-INTO stays allowed: adopting an +operator's old hygiene tag into the system row is the intended move. +""" +import pytest +from sqlalchemy import select + +from backend.app.models import Tag, TagKind + +pytestmark = pytest.mark.integration + +SYSTEM_NAMES = {"wip", "banner", "editor screenshot"} + + +async def _system_tag(db, name): + return (await db.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == name) + )).scalar_one() + + +@pytest.mark.asyncio +async def test_seeded_system_tags_exist(db): + rows = (await db.execute( + select(Tag.name, Tag.kind).where(Tag.is_system.is_(True)) + )).all() + assert {r.name for r in rows} == SYSTEM_NAMES + assert all(r.kind == TagKind.general for r in rows) + + +@pytest.mark.asyncio +async def test_get_tag_serializes_is_system(client, db): + wip = await _system_tag(db, "wip") + resp = await client.get(f"/api/tags/{wip.id}") + assert resp.status_code == 200 + assert (await resp.get_json())["is_system"] is True + + +@pytest.mark.asyncio +async def test_rename_system_tag_refused(client, db): + wip = await _system_tag(db, "wip") + resp = await client.patch( + f"/api/tags/{wip.id}", json={"name": "work in progress"} + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert "system tag" in body["error"] + + +@pytest.mark.asyncio +async def test_merge_away_system_tag_refused(client, db): + banner = await _system_tag(db, "banner") + created = await client.post( + "/api/tags", json={"name": "ad banner", "kind": "general"} + ) + target_id = (await created.get_json())["id"] + resp = await client.post( + f"/api/tags/{banner.id}/merge", json={"target_id": target_id} + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert "system tag" in body["error"] + + +@pytest.mark.asyncio +async def test_merge_into_system_tag_allowed(client, db): + wip = await _system_tag(db, "wip") + created = await client.post( + "/api/tags", json={"name": "old wip", "kind": "general"} + ) + source_id = (await created.get_json())["id"] + resp = await client.post( + f"/api/tags/{source_id}/merge", json={"target_id": wip.id} + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["target"]["id"] == wip.id + assert body["source_deleted"] is True diff --git a/tests/test_tag_query.py b/tests/test_tag_query.py index c5ed4f7..4719bf6 100644 --- a/tests/test_tag_query.py +++ b/tests/test_tag_query.py @@ -10,20 +10,29 @@ def test_serialize_tag_with_enum_kind(): # ORM/column rows carry kind as a TagKind enum. row = SimpleNamespace( id=1, name="Sasuke Uchiha", kind=TagKind.character, - fandom_id=5, fandom_name="Naruto", + fandom_id=5, fandom_name="Naruto", is_system=False, ) assert serialize_tag(row) == { "id": 1, "name": "Sasuke Uchiha", "kind": "character", - "fandom_id": 5, "fandom_name": "Naruto", + "fandom_id": 5, "fandom_name": "Naruto", "is_system": False, } def test_serialize_tag_with_string_kind_and_no_fandom(): - # autocomplete hits already store kind as a plain string. + # autocomplete hits already store kind as a plain string. A row from a + # select that predates the is_system column serializes as non-system. row = SimpleNamespace( id=2, name="solo", kind="general", fandom_id=None, fandom_name=None, ) assert serialize_tag(row) == { "id": 2, "name": "solo", "kind": "general", - "fandom_id": None, "fandom_name": None, + "fandom_id": None, "fandom_name": None, "is_system": False, } + + +def test_serialize_tag_system_row(): + row = SimpleNamespace( + id=3, name="wip", kind=TagKind.general, + fandom_id=None, fandom_name=None, is_system=True, + ) + assert serialize_tag(row)["is_system"] is True From e6f128c894ae5db7026423a2b1f81c31d28a3136 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 23:19:41 -0400 Subject: [PATCH 5/9] =?UTF-8?q?feat(ml):=20training=20hygiene=20=E2=80=94?= =?UTF-8?q?=20system-tagged=20images=20are=20absent=20from=20other=20conce?= =?UTF-8?q?pts=20training?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of milestone #128. _hygiene_excluded_ids (training_data.py) is the one shared predicate: images carrying any system tag are dropped from every OTHER concepts head training — not positives (a rough wip tagged as a character drags the head toward generic-sketch) and not rejection or sampled negatives (a wip OF character X is not evidence against X). A system tags own head trains on them unfiltered; that is what makes auto-flagging banners work. Selection is split out of train_head as the sklearn-free head_training_ids so CI (no sklearn) can pin the behavior. CCIP: reference prototypes skip hygiene-tagged images — a faceless wip figure region must never become an identity reference — and the ref cache signature now counts hygiene applications, since tagging an image wip changes the reference set without touching character/region counts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/services/ml/ccip.py | 27 ++++- backend/app/services/ml/heads.py | 61 ++++++++--- backend/app/services/ml/training_data.py | 25 ++++- tests/test_training_hygiene.py | 125 +++++++++++++++++++++++ 4 files changed, 223 insertions(+), 15 deletions(-) create mode 100644 tests/test_training_hygiene.py diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 2912ccd..8c98edd 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -62,6 +62,18 @@ def _single_character_images(): ) +def _hygiene_tagged_images(): + """Subquery of image ids carrying any SYSTEM tag (wip / banner / editor + screenshot). Training hygiene (#128): such images never contribute + reference prototypes — a faceless wip's figure region would otherwise + become an identity reference for the character it's tagged with.""" + return ( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ) + + async def _ref_signature(session: AsyncSession) -> tuple: n_tags = ( await session.execute( @@ -79,7 +91,17 @@ async def _ref_signature(session: AsyncSession) -> tuple: ) ) ).one() - return (n_tags, n_regs, max_id) + # Hygiene applications must invalidate too: tagging an image `wip` changes + # the reference set without touching character-tag or region counts. + n_hygiene = ( + await session.execute( + select(func.count()) + .select_from(image_tag) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ) + ).scalar_one() + return (n_tags, n_regs, max_id, n_hygiene) async def character_references(session: AsyncSession) -> dict[int, list]: @@ -102,6 +124,9 @@ async def character_references(session: AsyncSession) -> dict[int, list]: .where(ImageRegion.kind.in_(_FIGURE_KINDS)) .where(ImageRegion.ccip_embedding.is_not(None)) .where(ImageRegion.image_record_id.in_(_single_character_images())) + .where( + ImageRegion.image_record_id.not_in(_hygiene_tagged_images()) + ) ) ).all() refs: dict[int, list] = {} diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index 2f47401..d7d8480 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -40,6 +40,7 @@ from ...models import ( from ...models.tag import image_tag from .training_data import ( _auto_apply_point, + _hygiene_excluded_ids, _ids_with_tag, _l2norm, _load_embeddings, @@ -150,11 +151,16 @@ def train_all_heads( embedding_version = _embedder_version(session) eligible = _eligible_tag_ids(session, cfg["min_positives"]) eligible_set = set(eligible) + # Computed once per run, not per head — the hygiene set is identical for + # every non-system concept. + hygiene = _hygiene_excluded_ids(session) trained = 0 skipped = 0 for i, tag_id in enumerate(eligible): try: - ok = train_head(session, tag_id, embedding_version, cfg, np) + ok = train_head( + session, tag_id, embedding_version, cfg, np, hygiene=hygiene + ) except Exception: log.exception("train_head failed for tag %d", tag_id) ok = False @@ -174,27 +180,56 @@ def train_all_heads( return {"n_trained": trained, "n_skipped": skipped} +def head_training_ids( + session: Session, tag_id: int, cfg: dict, hygiene: set[int] | None = None, +) -> tuple[list[int], list[int]] | None: + """Select (pos_ids, neg_ids) for one head. Split out of train_head and + kept sklearn-free so the hygiene exclusion is testable in the CI env + (sklearn only exists in the ml image). Returns None when the concept has + too few usable positives. + + Training hygiene (#128): images carrying a system tag are ABSENT from + every other concept's training — dropped as positives AND kept out of + the rejection/sampled negative pool (see _hygiene_excluded_ids). A system + tag's own head trains on them unfiltered: its positives ARE the hygiene + images.""" + tag = session.get(Tag, tag_id) + if tag is not None and tag.is_system: + hygiene = set() + elif hygiene is None: + hygiene = _hygiene_excluded_ids(session) + + pos_ids = [i for i in _ids_with_tag(session, tag_id) if i not in hygiene] + if len(pos_ids) < cfg["min_positives"]: + return None + + pos_set = set(pos_ids) + rejected = [ + i for i in _rejected_ids(session, tag_id) + if i not in pos_set and i not in hygiene + ] + want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4) + sampled = _sample_unlabeled( + session, pos_set | set(rejected) | hygiene, + min(_UNLABELED_POOL, want_neg), + ) + return pos_ids, rejected + [i for i in sampled if i not in pos_set] + + def train_head( - session: Session, tag_id: int, embedding_version: str, cfg: dict, np + session: Session, tag_id: int, embedding_version: str, cfg: dict, np, + hygiene: set[int] | None = None, ) -> bool: """Fit + upsert one head. Returns True if a head was written, False if the concept had too few usable examples to train (the row is then removed).""" from sklearn.linear_model import LogisticRegression from sklearn.model_selection import StratifiedKFold, cross_val_predict - pos_ids = _ids_with_tag(session, tag_id) - if len(pos_ids) < cfg["min_positives"]: + ids = head_training_ids(session, tag_id, cfg, hygiene) + if ids is None: session.execute(delete(TagHead).where(TagHead.tag_id == tag_id)) return False - - pos_set = set(pos_ids) - rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set] - want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4) - sampled = _sample_unlabeled( - session, pos_set | set(rejected), min(_UNLABELED_POOL, want_neg) - ) - neg_ids = rejected + [i for i in sampled if i not in pos_set] - + pos_ids, neg_ids = ids emb = _load_embeddings(session, pos_ids + neg_ids) pos = [emb[i] for i in pos_ids if i in emb] neg = [emb[i] for i in neg_ids if i in emb] diff --git a/backend/app/services/ml/training_data.py b/backend/app/services/ml/training_data.py index d01acf7..5e2963a 100644 --- a/backend/app/services/ml/training_data.py +++ b/backend/app/services/ml/training_data.py @@ -17,10 +17,33 @@ from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session -from ...models import ImageRecord, TagSuggestionRejection +from ...models import ImageRecord, Tag, TagSuggestionRejection from ...models.tag import image_tag +def _hygiene_excluded_ids(session: Session) -> set[int]: + """Ids of images carrying ANY system tag (wip / banner / editor + screenshot — milestone #128). These images are excluded from OTHER + concepts' head training entirely: not positives (a rough wip tagged as a + character drags that head toward 'generic sketch') and not sampled or + rejection negatives (a wip OF character X is not evidence against X) — + simply absent. A system tag's OWN head trains on them unchanged; that is + what makes auto-flagging banners/editor screenshots work. + + Item-level by design: a wip-tagged process video contributes (or + withholds) ALL its sampled frames, though some may show the finished + piece. Operator call 2026-07-03: with enough clean data this washes out — + no per-frame handling. + """ + return set( + session.execute( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.is_system.is_(True)) + ).scalars().all() + ) + + def _ids_with_tag(session: Session, tag_id: int) -> list[int]: return [ r[0] for r in session.execute( diff --git a/tests/test_training_hygiene.py b/tests/test_training_hygiene.py new file mode 100644 index 0000000..91be7d4 --- /dev/null +++ b/tests/test_training_hygiene.py @@ -0,0 +1,125 @@ +"""Training hygiene (#128): system-tagged images are ABSENT from other +concepts' training data and from CCIP reference prototypes. + +sklearn only exists in the ml image, so these pin head_training_ids (the +sklearn-free selection split out of train_head) rather than a full fit — +the exclusion lives entirely in that selection. +""" +import pytest +from sqlalchemy import insert, select + +from backend.app.models import ImageRecord, ImageRegion, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.ml import ccip +from backend.app.services.ml.heads import head_training_ids +from backend.app.services.ml.training_data import _hygiene_excluded_ids +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_CFG = {"min_positives": 2, "neg_ratio": 1} + + +async def _system_wip(db) -> Tag: + return (await db.execute( + select(Tag).where(Tag.is_system.is_(True), Tag.name == "wip") + )).scalar_one() + + +async def _img(db, sha, *, embedded=True): + rec = ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", + integrity_status="unknown", + siglip_embedding=([0.1] * 1152 if embedded else None), + ) + db.add(rec) + await db.flush() + return rec + + +async def _apply(db, image_id, tag_id): + await db.execute(insert(image_tag).values( + image_record_id=image_id, tag_id=tag_id, source="manual", + )) + + +@pytest.mark.asyncio +async def test_hygiene_excluded_ids(db): + wip = await _system_wip(db) + flagged = await _img(db, "a" * 64) + await _img(db, "b" * 64) + await _apply(db, flagged.id, wip.id) + excluded = await db.run_sync(lambda s: _hygiene_excluded_ids(s)) + assert excluded == {flagged.id} + + +@pytest.mark.asyncio +async def test_head_selection_drops_hygiene_from_both_sides(db): + """A wip-tagged image of concept X is neither a positive NOR a sampled + negative for X — it is absent entirely.""" + wip = await _system_wip(db) + concept = await TagService(db).find_or_create("concept_x", TagKind.general) + clean_a = await _img(db, "a" * 64) + clean_b = await _img(db, "b" * 64) + flagged_pos = await _img(db, "c" * 64) # X + wip: dropped positive + negative_pool = await _img(db, "d" * 64) # untagged: legit negative + flagged_pool = await _img(db, "e" * 64) # wip only: must not be sampled + for img in (clean_a, clean_b, flagged_pos): + await _apply(db, img.id, concept.id) + await _apply(db, flagged_pos.id, wip.id) + await _apply(db, flagged_pool.id, wip.id) + + ids = await db.run_sync(lambda s: head_training_ids(s, concept.id, _CFG)) + assert ids is not None + pos_ids, neg_ids = ids + assert set(pos_ids) == {clean_a.id, clean_b.id} + assert negative_pool.id in neg_ids + assert flagged_pos.id not in neg_ids + assert flagged_pool.id not in neg_ids + + +@pytest.mark.asyncio +async def test_system_tags_own_head_keeps_hygiene_positives(db): + """The wip head itself trains ON wip-tagged images — that's what makes + auto-flagging work.""" + wip = await _system_wip(db) + one = await _img(db, "a" * 64) + two = await _img(db, "b" * 64) + await _img(db, "c" * 64) # negative pool + await _apply(db, one.id, wip.id) + await _apply(db, two.id, wip.id) + + ids = await db.run_sync(lambda s: head_training_ids(s, wip.id, _CFG)) + assert ids is not None + pos_ids, _ = ids + assert set(pos_ids) == {one.id, two.id} + + +@pytest.mark.asyncio +async def test_ccip_references_skip_hygiene_images(db): + """A wip's figure region must never become an identity prototype, even on + a single-character image.""" + ccip._REF_CACHE.update(sig=None, refs=None) + wip = await _system_wip(db) + char = await TagService(db).find_or_create("Char A", TagKind.character) + + clean = await _img(db, "a" * 64) + flagged = await _img(db, "b" * 64) + for img, slot in ((clean, 0), (flagged, 1)): + vec = [0.0] * 768 + vec[slot] = 1.0 + db.add(ImageRegion( + image_record_id=img.id, kind="figure", + rx=0.0, ry=0.0, rw=1.0, rh=1.0, + ccip_embedding=vec, embedding_version="ccip-test", + )) + await _apply(db, img.id, char.id) + await _apply(db, flagged.id, wip.id) + await db.commit() + + refs = await ccip.character_references(db) + vectors = refs.get(char.id, []) + assert len(vectors) == 1 + assert float(vectors[0][0]) == pytest.approx(1.0) + assert float(vectors[0][1]) == pytest.approx(0.0) From 19744fa41d02ccc3ec648084080c851aab8ea4ac Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 23:21:52 -0400 Subject: [PATCH 6/9] fix(tests): resync serial sequences after baseline restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TRUNCATE ... RESTART IDENTITY resets every sequence to 1, and the baseline restore re-inserts seeded rows WITH their explicit ids — leaving each sequence pointing below MAX(id). Harmless while the only baseline rows lived in tables tests never sequence-insert into (ml_settings id=1); migration 0075 seeded tag rows and every Tag insert after the first truncate collided on pk_tag id=1 (205 failures, run 1888 — find_or_create then surfaced it as NoResultFound via its conflict-recovery re-select). setval every restored table with a serial id column past its restored rows. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- tests/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b8733e0..be692d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -166,6 +166,17 @@ def _reset_db_after_integration(request, _truncate_engine): rows = (_SEED_SNAPSHOT or {}).get(t.name) if rows: conn.execute(t.insert(), rows) + # Restored rows carry their explicit ids while RESTART + # IDENTITY reset the sequence to 1, so the next ORM insert + # would collide on the PK (bitten by migration 0075's seeded + # system tags: every Tag insert failed with pk_tag id=1). + # Resync any serial id sequence past the restored rows. + if "id" in t.c: + conn.exec_driver_sql( + f"SELECT setval(pg_get_serial_sequence('{t.name}', 'id'), " + f"(SELECT COALESCE(MAX(id), 1) FROM {t.name})) " + f"WHERE pg_get_serial_sequence('{t.name}', 'id') IS NOT NULL" + ) @pytest_asyncio.fixture(autouse=True) From 723f023e6af63f735fa2b8f763b30bcbcde9bc27 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 23:23:30 -0400 Subject: [PATCH 7/9] feat(gallery): similar() hides presentation images (banner / editor screenshot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 of milestone #128. Presentation-tagged images cluster on UI chrome rather than content, so near any one of them they fill the whole more-like-this grid. Excluded from candidates in the ONE whole-image similarity surface (gallery similar mode, explore walk, and RelatedStrip all ride GalleryService.similar) — the anchor itself may be a banner, and wip stays surfaced: only the training pipelines exclude it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/models/tag.py | 6 ++++++ backend/app/services/gallery_service.py | 16 ++++++++++++++- tests/test_gallery_similar.py | 26 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index fe02dcd..2479578 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -43,6 +43,12 @@ class TagKind(StrEnum): # to keep historic tag rows queryable. +# The seeded system tags (migration 0075). PRESENTATION tags additionally +# hide from whole-image similarity results — they cluster on UI chrome, not +# content. `wip` is real art: only the training pipelines exclude it. +SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") +PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot") + image_tag = Table( "image_tag", Base.metadata, diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index e385459..4cde351 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag -from ..models.tag import image_tag +from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag from .pagination import decode_cursor, encode_cursor from .tag_query import ( fandom_join_alias, @@ -693,9 +693,23 @@ class GalleryService: eff = _effective_date_col() stmt = select(ImageRecord, Post.post_date, eff.label("eff")) stmt = _outer_join_primary_post(stmt) + # Presentation images (banner / editor-screenshot system tags, #128) + # cluster on UI chrome rather than content, so near any one of them + # they'd fill the grid. Excluded from CANDIDATES only — the anchor + # itself may be a banner, and `wip` stays surfaced (real art; only + # the training pipelines exclude it). + presentation = ( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where( + Tag.is_system.is_(True), + Tag.name.in_(PRESENTATION_SYSTEM_TAGS), + ) + ) stmt = stmt.where( ImageRecord.siglip_embedding.is_not(None), ImageRecord.id != image_id, + ImageRecord.id.not_in(presentation), ) stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=None, diff --git a/tests/test_gallery_similar.py b/tests/test_gallery_similar.py index ea9b355..35e75d6 100644 --- a/tests/test_gallery_similar.py +++ b/tests/test_gallery_similar.py @@ -4,6 +4,7 @@ precomputed SigLIP image embeddings. No query-time ML inference.""" from datetime import UTC, datetime, timedelta import pytest +from sqlalchemy import select from backend.app.models import ImageRecord, Tag, TagKind from backend.app.models.tag import image_tag @@ -72,6 +73,31 @@ async def test_similar_missing_source_returns_none(db): assert await svc.similar(99999, limit=10) is None +@pytest.mark.asyncio +async def test_similar_excludes_presentation_tagged_images(db): + """banner / editor-screenshot system tags (#128) hide from similar + RESULTS — they cluster on UI chrome, not content. A banner anchor still + gets results; `wip`-tagged images stay in (real art).""" + src = await _img(db, 1, _vec(1, 0)) + bannered = await _img(db, 2, _vec(1, 0.02)) # nearest, but a banner + wipped = await _img(db, 3, _vec(1, 0.3)) + plain = await _img(db, 4, _vec(1, 0.6)) + banner_tag = (await db.execute(select(Tag).where( + Tag.is_system.is_(True), Tag.name == "banner"))).scalar_one() + wip_tag = (await db.execute(select(Tag).where( + Tag.is_system.is_(True), Tag.name == "wip"))).scalar_one() + await db.execute(image_tag.insert().values( + image_record_id=bannered.id, tag_id=banner_tag.id, source="manual")) + await db.execute(image_tag.insert().values( + image_record_id=wipped.id, tag_id=wip_tag.id, source="manual")) + svc = GalleryService(db) + res = await svc.similar(src.id, limit=10) + assert [i.id for i in res] == [wipped.id, plain.id] + # A presentation-tagged ANCHOR still answers — only candidates hide. + res_from_banner = await svc.similar(bannered.id, limit=10) + assert {i.id for i in res_from_banner} == {src.id, wipped.id, plain.id} + + @pytest.mark.asyncio async def test_similar_composes_with_tag_filter(db): src = await _img(db, 1, _vec(1, 0)) From f77e75147dfee1f72e7725fafd2b98e336d879b6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Jul 2026 08:34:16 -0400 Subject: [PATCH 8/9] feat(tags): system-tag UI markers + full protection sweep (step 4 of #128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UI: shield marker + tooltip on TagChip and TagCard; system tags hide rename/merge/delete affordances (chip kebab entirely — set-fandom never applies to their general kind; remove stays, un-tagging is normal use). Aliases stay available: mapping model outputs ONTO a system tag is useful. Directory cards carry is_system. Every destructive path that could take out a system row is now guarded, found by sweeping run 1891s off-by-three failures — each one was a surface that would have eaten the seeded tags: - prune-unused: predicate exempts is_system (they ship with zero applications and matched every unused condition) - reset-content: predicate exempts is_system AND keeps their applications — hygiene flags describe the file, not content tagging - admin tag DELETE: refused with system_tag error - normalize_existing_tags: scan excludes is_system — canonicalization would recase wip -> Wip behind TagService.rename's guard, breaking the name-keyed presentation lookup Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/api/admin.py | 4 + backend/app/services/cleanup_service.py | 16 +++- backend/app/services/tag_directory_service.py | 1 + backend/app/services/tag_service.py | 4 + frontend/src/components/discovery/TagCard.vue | 13 ++++ frontend/src/components/modal/TagChip.vue | 16 +++- tests/test_system_tags.py | 75 ++++++++++++++++++- 7 files changed, 121 insertions(+), 8 deletions(-) diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index d94615b..e1ef468 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -156,6 +156,10 @@ async def tag_delete(tag_id: int): ) except LookupError: return _bad("not_found", status=404) + except ValueError as exc: + # System tags (#128) — the training-hygiene machinery keys on + # these rows. + return _bad("system_tag", detail=str(exc)) return jsonify(result) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 8708e28..cd7ef99 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -from sqlalchemy import delete, func, or_, select, update +from sqlalchemy import and_, delete, func, or_, select, update from sqlalchemy.orm import Session, aliased from ..models import ( @@ -203,6 +203,9 @@ def _unused_tag_conditions() -> list: Tag.id.not_in(used_via_series), Tag.id.not_in(used_via_chapter), Tag.id.not_in(used_via_fandom), + # System tags (#128) ship with zero applications and must survive a + # prune — the training-hygiene machinery keys on the rows. + Tag.is_system.is_(False), ] @@ -402,6 +405,8 @@ def delete_tag(session: Session, *, tag_id: int) -> dict: tag = session.get(Tag, tag_id) if tag is None: raise LookupError(f"tag id not found: {tag_id}") + if tag.is_system: + raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted") associations_count = count_tag_associations(session, tag_id=tag_id) info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value} session.delete(tag) @@ -731,7 +736,10 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict: heads retrain from whatever the operator re-tags. (The API route gates the live run behind a preview-derived confirm token for exactly this reason.) - PRESERVED: fandom + series tags and their series_page ordering. CASCADE on + PRESERVED: fandom + series tags and their series_page ordering, AND the + system hygiene tags (#128) WITH their applications — the reset re-tags + CONTENT concepts, while wip/banner flags describe the file itself and + re-flagging hundreds of banners by hand would be pure loss. CASCADE on image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting character tags never touches the fandom rows. Irreversible except via DB backup @@ -744,7 +752,9 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict: "sample_names": [first 50], and on live runs "deleted": total} """ - predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS) + predicate = and_( + Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False) + ) rows = session.execute( select(Tag.id, Tag.name, Tag.kind).where(predicate) ).all() diff --git a/backend/app/services/tag_directory_service.py b/backend/app/services/tag_directory_service.py index 49c55cc..a6b3756 100644 --- a/backend/app/services/tag_directory_service.py +++ b/backend/app/services/tag_directory_service.py @@ -107,6 +107,7 @@ class TagDirectoryService: "kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind, "fandom_id": tag.fandom_id, "fandom_name": fandom_name, + "is_system": tag.is_system, "image_count": int(image_count), "preview_thumbnails": previews.get(tag.id, []), } diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index 3bda769..98e1d9c 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -806,6 +806,10 @@ async def normalize_existing_tags( rows = ( await session.execute( select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id) + # System tags (#128) are exempt: their names are the keys the + # hygiene machinery matches on, and canonicalization would recase + # 'wip' → 'Wip' (this path bypasses TagService.rename's guard). + .where(Tag.is_system.is_(False)) ) ).all() groups = _group_existing_tags(rows) diff --git a/frontend/src/components/discovery/TagCard.vue b/frontend/src/components/discovery/TagCard.vue index 1d1a0b1..9aa7582 100644 --- a/frontend/src/components/discovery/TagCard.vue +++ b/frontend/src/components/discovery/TagCard.vue @@ -13,9 +13,15 @@