diff --git a/backend/app/services/post_naming.py b/backend/app/services/post_naming.py new file mode 100644 index 0000000..da46271 --- /dev/null +++ b/backend/app/services/post_naming.py @@ -0,0 +1,319 @@ +"""The creator's own working name for a piece, recovered from a filename. + +Milestone 388. Pure functions, no DB and no ML — the whole point is that this +signal is free. + +## What this is for + +Two of the operator's artists post a censored or cropped teaser on Patreon and +the real release in their Discord. Matching those by IMAGE is the pair a +whole-image comparison handles worst: the teaser is a crop with a censor bar, +which is exactly the local edit that moves a perceptual hash and blurs a +semantic embedding. + +But the creator names both exports after the same internal working title, and +that name survives into both platforms untouched. Measured on the live instance +2026-09-24, artist 8: + + 01_((0-k <-> 0-k_base (1.3h apart) + 01_680LC <-> 680LC_Border (21.0h apart) + 01_cnni18x <-> cnni18x (21.5h apart) + +Three pairs, no false positives, and **two of them are 21 hours apart** — far +enough that time proximity scores them ~0.10 and could never propose them. The +naming signal is orthogonal to the timing one: each finds pairs the other +cannot, which is why both are kept rather than one being tuned to cover both. + +## Why a filename and not a perceptual hash + +A shared working-name token is IDENTITY evidence — `680lc` appearing on both +platforms is not a coincidence. Proximity is CIRCUMSTANTIAL: it says two things +happened near each other, never that they are the same thing. The distinction +drives the weighting in `post_association_service`, and it is why a rare enough +token is allowed to carry a proposal on its own while no amount of circumstance +is. + +## The one false-positive class found, and why the fix is shaped this way + +A first pass matched `01_Screenshot 2026-08-13 000004` to +`Screenshot_2026-08-13_032144` on the token `2026-08-13`, twice. + +A screenshot filename is a camera artifact. It carries no working name, and the +date inside it collides across platforms on the same day BY CONSTRUCTION — the +teaser and the release are posted the same day, so their screenshot names +always share a date token. That is a signal that fires precisely when it is +least informative. + +So a filename with no working name contributes NOTHING, rather than the +plausible-looking date match it could be squeezed for. Re-run with that rule: +the same three true pairs, zero false. Half of this creator's recent teasers +are screenshots, and those pairs are simply out of this signal's reach — which +is where crop-to-source matching earns its cost, and nowhere else. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable +from pathlib import PurePosixPath + +# A screenshot name, on either platform. Patreon's importer writes +# `01_Screenshot 2026-08-13 000004`; gallery-dl's Discord naming writes +# `Screenshot_2026-09-22_003651`. Matched after the index/message prefixes are +# stripped, so both shapes reach this as a bare `Screenshot ...`. +# NOT `\b` after "shot": `\b` needs a word/non-word transition and `_` is a +# WORD character, so `Screenshot_2026-08-13_032144` — gallery-dl's Discord +# spelling — sailed straight past the guard while the space-separated Patreon +# spelling was caught. Found by running this against the live library rather +# than by reading it. Assert the next character is not a letter instead. +_SCREENSHOT = re.compile(r"^screen[ _-]?shot(?![a-z])", re.I) + +# The importer's per-post media index: `01_`, `02_`. Not part of any name. +_MEDIA_INDEX = re.compile(r"^\d{1,3}_") + +# gallery-dl's Discord filename pattern (#3999): +# `{date:%Y%m%d}_{message_id}_{num:>02}_{filename}`. +_DISCORD_PREFIX = re.compile(r"^\d{8}_\d{6,}_\d{1,3}_") + +# The importer's content-hash suffix, `__<10 hex>`, sometimes doubled on files +# that went through an older import era. +_HASH_SUFFIX = re.compile(r"(?:__[0-9a-f]{10})+$") + +# Generic export decorations. Stripped as SUFFIXES so the stem survives: +# `cnni18x_wip3` and `cnni18x` must yield the same token, or a work-in-progress +# would never match the piece it became. +_DECORATION = re.compile( + r"(?:[_-]?(?:wip|base|final|alt|alts|edit|edits|border|clean|raw|hd|full|" + r"censored|uncensored|nsfw|sfw|ver|v)\d*)+$", + re.I, +) + +# Tokens that carry no identity even when they survive the rules above. +_STOPWORDS = frozenset({ + "img", "image", "untitled", "new", "test", "page", "final", "copy", + "post", "media", "file", "avatar", "cover", "banner", "icon", "splash", + # Literally the string "None": issue #3999's Discord naming rendered + # `{user[name]}` as it for ~1,600 files, so it is the single most common + # "name" in the library and identifies nothing. + "none", +}) + +# A bare year, or a date fragment. These are what made the screenshot collision +# look like a match, and they are worthless as identity even in a real name. +_YEAR = re.compile(r"^(?:19|20)\d{2}$") +_ALL_DIGITS = re.compile(r"^\d+$") + +MIN_TOKEN_LEN = 3 + +# A token shared by more than this many of ONE ARTIST's images is a habit, not +# an identity — a character name, a series tag, a recurring export preset. The +# bar is deliberately low: a working name identifies one piece, so it should +# appear on that piece's handful of exports (base, wips, the teaser crop) and +# nowhere else. Raising this trades false positives for reach, which is the +# wrong direction here — a wrong link asserts two different pieces are one. +MAX_TOKEN_FREQUENCY = 6 + + +def _strip_prefixes(stem: str) -> str: + """Remove the framing each platform's importer adds around the real name.""" + stem = _DISCORD_PREFIX.sub("", stem) + stem = _MEDIA_INDEX.sub("", stem) + return _HASH_SUFFIX.sub("", stem) + + +def working_name_tokens(path: str) -> set[str]: + """The identity-bearing tokens in one image's filename. + + Returns an EMPTY set for a name that carries no working title — a + screenshot, a bare number, a stopword. Empty means "no evidence", which the + caller must treat as silence rather than as a weak match; see the module + docstring for the false positive that rule exists for. + """ + stem = _strip_prefixes(PurePosixPath(path).stem) + if _SCREENSHOT.match(stem.strip()): + return set() + + out: set[str] = set() + # Hyphens are kept INSIDE tokens — `0-k` is a real working name on the live + # instance, and splitting on hyphen would reduce it to a single character + # and then discard it for being too short. + for raw in re.split(r"[^0-9A-Za-z-]+", stem.lower()): + tok = _DECORATION.sub("", raw).strip("-") + if len(tok) < MIN_TOKEN_LEN: + continue + if tok in _STOPWORDS or _YEAR.match(tok) or _ALL_DIGITS.match(tok): + continue + out.add(tok) + return out + + +def token_frequencies(paths: Iterable[str]) -> Counter[str]: + """How often each working-name token appears across one artist's images. + + Scoped to the ARTIST, not the library: a working name belongs to the person + who chose it, and the same string can be one creator's piece and another's + boilerplate. Built once per artist per sweep, not per candidate pair. + """ + counts: Counter[str] = Counter() + for p in paths: + counts.update(working_name_tokens(p)) + return counts + + +def _rarity(freq: int, max_frequency: int) -> float: + """Rarity of one token within an artist's own corpus, in [0, 1]. + + Shared by BOTH signals deliberately. They carried one formula each + until 2026-09-24, and the copies drifted: the filename signal grew a + frequency gate and the marker signal never did, so a creator's habitual + emoji scored the same 1.00 as a marker they had used twice. One + definition cannot drift from itself. + + Full strength at 2 rather than 1: a genuine match means the token is on + at least two things, so demanding uniqueness would reject every real + pair. Decays to zero AT the cap rather than falling off it, so nothing + sits on a cliff edge. + """ + if freq <= 2: + return 1.0 + if freq >= max_frequency: + return 0.0 + return (max_frequency - freq) / (max_frequency - 2) + + +def shared_identity( + left: Iterable[str], + right: Iterable[str], + frequencies: Counter[str], + *, + max_frequency: int = MAX_TOKEN_FREQUENCY, +) -> tuple[float, str | None]: + """Strength in [0, 1] that two sets of filenames name the SAME piece. + + Returns `(strength, token)` — the token is carried back so the proposal can + say WHY it was made. A review queue that cannot explain itself is one the + operator learns to click through without reading. + + Strength is a function of the winning token's rarity within the artist's + own library, not of how many tokens matched. One decisive token beats three + vague ones, and a token that appears on forty of this artist's images is a + habit rather than an identity however exactly it matches. + """ + shared = {t for t in set(left) & set(right) if frequencies.get(t, 0) <= max_frequency} + if not shared: + return 0.0, None + + # The rarest shared token decides — one decisive token beats three vague + # ones. `frequencies` counts IMAGES here, and a real working name lands on + # a few of them: the base, its wips, the teaser crop. + token = min(shared, key=lambda t: (frequencies.get(t, 0), -len(t), t)) + strength = round(_rarity(max(frequencies.get(token, 1), 1), max_frequency), 4) + # A token sitting exactly ON the cap decays to zero, and naming it anyway + # would hand the review queue a reason that carries no weight — "matched on + # loislanetb2", with nothing behind it. Measured: that token is on 6 of this + # artist's images. Report a token only when it is doing work. + return (strength, token) if strength > 0 else (0.0, None) + + +# --- the body/title signal --------------------------------------------------- +# +# The same idea applied to TEXT. The operator's example pair carries `🍈🍈` in +# the Patreon title and `@everyone 🍈 🍈` in the Discord message — a marker the +# creator uses to tie the two together, which no vocabulary list would predict. +# +# Rarity-gated, exactly as the filename signal is, and the gate is here because +# the first pass did NOT have one. Measured on artist 8, 300 posts: +# +# 💦 11 posts (4%) 🫴 6 🌰 5 🍗 5 🫣 4 +# +# 💦 is punctuation for this creator — about one post in twenty-five. Ungated it +# scored a full 1.00 and was the DECIDING term in a proposal that proximity +# alone (0.441) could not carry. A habitual marker riding along with proximity +# is just proximity wearing a hat, which is the exact failure the matcher's +# threshold sits above 0.55 to prevent. The operator's 🍈🍈 is the opposite +# case: two posts, and they are the pair itself. + +_WORD = re.compile(r"[0-9A-Za-z]{3,}") +# Anything outside the Basic Multilingual Plane's text ranges: emoji, symbols, +# kaomoji parts. These are the tokens creators actually use as markers, and +# they are rare enough in prose to be evidence on their own. +# U+1F000-1FAFF is the emoji planes; U+2190-2BFF covers arrows, dingbats and +# the miscellaneous-symbol blocks, which already contains U+2600-27BF. +_SYMBOL = re.compile(r"[\U0001F000-\U0001FAFF\u2190-\u2BFF]") + +_COMMON_TEXT = frozenset({ + "the", "and", "for", "you", "new", "out", "now", "this", "that", "with", + "everyone", "here", "post", "all", "art", "one", "get", "has", "are", +}) + +# A marker in more than this many of ONE ARTIST's posts is a signature, not a +# tie-back. Tighter than MAX_TOKEN_FREQUENCY because the units differ and so +# does the evidence: that one counts a working name across a piece's handful of +# EXPORTS, where this counts a public decoration across POSTS. A marker that +# ties an announcement to its drop lands on two posts — the two. +MAX_MARKER_FREQUENCY = 4 + + +def text_markers(text: str | None) -> set[str]: + """Distinctive tokens in a post body or title: symbols, and rare-ish words. + + Symbols count individually rather than as a run, so `🍈🍈` and `🍈 🍈` — + which is how the same marker appears on the two platforms — reduce to the + same token. Spacing is a platform's rendering, not the creator's intent. + """ + if not text: + return set() + out = {m.group(0) for m in _SYMBOL.finditer(text)} + out |= { + w.lower() for w in _WORD.findall(text) + if w.lower() not in _COMMON_TEXT and not _YEAR.match(w) + } + return out + + +def marker_frequencies(texts: Iterable[str | None]) -> Counter[str]: + """How many of ONE ARTIST's posts each marker appears in. + + Per POST, not per occurrence: a creator who repeats an emoji six times in + one body has used it once as far as identity goes. Scoped to the artist for + the same reason `token_frequencies` is — a marker is a personal habit, and + one creator's signature is another's whole vocabulary. + """ + counts: Counter[str] = Counter() + for t in texts: + counts.update(text_markers(t)) + return counts + + +def marker_overlap( + left: str | None, + right: str | None, + frequencies: Counter[str], + *, + max_frequency: int = MAX_MARKER_FREQUENCY, +) -> float: + """Strength in [0, 1] that two texts share a DELIBERATE marker. + + `frequencies` is required rather than defaulted to "no gate". An ungated + call is the bug this signature exists to make impossible to write by + accident, and a default would have kept it one keyword away. + + Symbols weigh full and words a quarter, because prose shares words by + accident: a creator who writes "commission" in both posts on a Tuesday has + told us nothing that the timestamps did not already say. + + There is no divisor. An earlier pass halved the total so that a long body + could not out-vote a short one, which the rarity gate now does properly — + and halving meant the operator's own 🍈🍈 pair, a marker on exactly two + posts, could reach only 0.5. One marker the creator uses nowhere else is + the whole signal, not half of it. + """ + shared = text_markers(left) & text_markers(right) + if not shared: + return 0.0 + score = sum( + (1.0 if _SYMBOL.match(t) else 0.25) * _rarity(frequencies.get(t, 1), max_frequency) + for t in shared + ) + return round(min(1.0, score), 4) diff --git a/tests/test_post_naming.py b/tests/test_post_naming.py new file mode 100644 index 0000000..1a6c545 --- /dev/null +++ b/tests/test_post_naming.py @@ -0,0 +1,233 @@ +"""Milestone 388: the creator's own working name, recovered from a filename. + +Two of the operator's artists post a censored or cropped teaser on Patreon and +the real release in their Discord. That pair is the one a whole-image +comparison handles WORST — a crop with a censor bar is exactly the local edit +that moves a perceptual hash and blurs a semantic embedding — and it is the +pair the creator names identically on both platforms. + +Everything here was calibrated against the live library (artist 8, 520 images, +409 distinct tokens) rather than invented, so the cases carry their measured +numbers. The property the whole module serves is that a WRONG link is worse +than no link: no link leaves the operator where they already were, a wrong one +tells them two different pieces are the same piece. +""" +from collections import Counter + +import pytest + +from backend.app.services.post_naming import ( + MAX_MARKER_FREQUENCY, + MAX_TOKEN_FREQUENCY, + marker_frequencies, + marker_overlap, + shared_identity, + text_markers, + token_frequencies, + working_name_tokens, +) + +# --- what survives each platform's framing ----------------------------------- +# +# Three naming eras reach this (#4002), and a token has to come out the same +# from all of them or the signal only works on whichever era it was written +# against. + + +@pytest.mark.parametrize( + "path, expected, era", + [ + ( + "/images/yellowroom/patreon/2026-09-23_170392790_Anya/01_((0-k.jpg", + "0-k", + "current Patreon: a per-post media index", + ), + ( + "20260923_1552424119805673604_01_0-k_base.jpg", + "0-k", + "Discord: gallery-dl's date_messageid_num_ prefix (#3999)", + ), + ( + "85317841_media_212565911_0071 NoHeart__c3118a69f3__c3118a69f3.jpg", + "noheart", + "legacy: flat at the artist root, post id + media id + doubled hash", + ), + ], +) +def test_one_working_name_survives_every_naming_era(path, expected, era): + assert expected in working_name_tokens(path), era + + +def test_a_hyphen_stays_inside_a_token(): + """`0-k` is a real working name on the live instance — it is the operator's + own example pair. Splitting on hyphen reduces it to two single characters + and then discards both for being too short, which silently loses the one + case this was built to catch.""" + assert working_name_tokens("01_((0-k.jpg") == {"0-k"} + + +def test_a_work_in_progress_matches_the_piece_it_became(): + """Measured: `cnni6600 wip1` (Patreon) and `cnni6600_Base` (Discord) are + one piece. A decoration is a stage, not a name.""" + assert working_name_tokens("01_cnni6600 wip1.png") == working_name_tokens( + "20250514_1372285581945471087_01_cnni6600_Base.jpg" + ) + + +def test_the_literal_string_none_is_not_a_name(): + """#3999 rendered `{user[name]}` as "None" for ~1,600 files, which made it + the single most common "name" in the library and an identity for nothing.""" + assert working_name_tokens("20180508_None_DCthingwhat__0123456789.jpg") == { + "dcthingwhat" + } + + +# --- the false positive that shaped the design ------------------------------- + + +@pytest.mark.parametrize( + "path", + [ + "01_Screenshot 2026-08-13 000004.png", # Patreon's spelling + "20260813_1234567890123_01_Screenshot_2026-08-13_032144.png", # Discord's + ], +) +def test_a_screenshot_contributes_nothing_at_all(path): + """A first pass matched these two to each other on the token `2026-08-13`. + + A screenshot filename is a camera artifact carrying no working name, and + its date collides across platforms BY CONSTRUCTION: the teaser and the + release go out the same day, so their screenshot names always share one. + That is a signal firing precisely when it is least informative. + + Both spellings matter. `\\b` after "shot" was the first guard and it caught + only the Patreon one, because `_` is a WORD character so + `Screenshot_2026-08-13` never presented a word boundary there. Found by + running against the live library, not by reading the regex. + """ + assert working_name_tokens(path) == set() + + +# --- rarity, which both signals now share ------------------------------------ + + +def test_a_token_on_two_images_is_full_strength(): + """Two is the floor rather than one: a genuine match means the token is on + at least two files, so demanding uniqueness rejects every real pair.""" + freqs = Counter({"connfront": 2}) + + assert shared_identity({"connfront"}, {"connfront"}, freqs) == (1.0, "connfront") + + +def test_a_character_name_can_never_link_two_posts(): + """THE false-positive guard, and the reason the cap is set where it is. + Measured on artist 8: `anya` is on 8 images, `riju` 9, `undyne` 8, `bea` 16. + Ungated, every Anya post would match every Anya drop.""" + freqs = Counter({"anya": 8}) + + assert shared_identity({"anya"}, {"anya"}, freqs) == (0.0, None) + + +def test_a_token_at_the_cap_names_nothing(): + """It decays to zero, and reporting it anyway would hand the review queue a + reason with no weight behind it — "matched on loislanetb2", with nothing + there. A token is named only while it is doing work.""" + freqs = Counter({"tok": MAX_TOKEN_FREQUENCY}) + + assert shared_identity({"tok"}, {"tok"}, freqs) == (0.0, None) + + +def test_the_rarest_shared_token_decides_not_the_count_of_them(): + """One decisive token beats three vague ones. A pair sharing a piece name + AND two habits is evidence of the piece name.""" + freqs = Counter({"rare": 2, "goo": 9, "shiny": 7}) + + strength, token = shared_identity( + {"rare", "goo", "shiny"}, {"rare", "goo", "shiny"}, freqs + ) + + assert (strength, token) == (1.0, "rare") + + +def test_frequencies_are_counted_per_artist_not_per_library(): + """A working name belongs to the person who chose it; the same string is + one creator's piece and another's boilerplate.""" + counts = token_frequencies( + ["01_ConnFront.jpg", "20230222_1078078245695664148_01_ConnFront.jpg"] + ) + + assert counts["connfront"] == 2 + + +# --- the text marker, gated the same way ------------------------------------- + + +def test_the_same_marker_reads_the_same_through_both_platforms(): + """The operator's pair carries `\U0001F348\U0001F348` in the Patreon title and + `@everyone \U0001F348 \U0001F348` in the Discord message. Spacing is a platform's + rendering, not the creator's intent.""" + assert "\U0001F348" in text_markers("Anya -- \U0001F348\U0001F348") + assert "\U0001F348" in text_markers("@everyone \U0001F348 \U0001F348") + + +def test_one_marker_the_creator_uses_nowhere_else_is_the_whole_signal(): + """Measured: `\U0001F317` is on exactly two of this artist's 300 posts, and they + are the pair. An earlier pass halved every total so a long body could not + out-vote a short one — which the rarity gate now does properly, and which + meant this case could reach only 0.5.""" + freqs = Counter({"\U0001F317": 2}) + + assert marker_overlap("\U0001F317", "\U0001F317 drop", freqs) == 1.0 + + +def test_a_marker_the_creator_uses_habitually_is_worth_nothing(): + """The bug this gate exists for. Measured on artist 8: \U0001F4A6 is in 13 of 300 + posts — punctuation, about one post in twenty-five. Ungated it scored a + full 1.00 and was the DECIDING term in a proposal that proximity alone + could not carry, which is proximity wearing a hat.""" + freqs = Counter({"\U0001F4A6": 13}) + + assert marker_overlap("Drizzle \U0001F4A6", "\U0001F4A6", freqs) == 0.0 + + +def test_a_shared_ordinary_word_is_worth_nothing(): + """Also measured, also a real proposal: the shared "marker" between a + teaser and a drop was the word `like`, which is in 43 of this artist's + posts.""" + freqs = Counter({"like": 43}) + + assert marker_overlap("like this", "like that", freqs) == 0.0 + + +def test_a_symbol_outweighs_a_word_at_equal_rarity(): + """Prose shares words by accident; a creator who writes "commission" in + both posts on a Tuesday has said nothing the timestamps did not.""" + freqs = Counter({"\U0001F317": 2, "commission": 2}) + + symbol = marker_overlap("\U0001F317", "\U0001F317", freqs) + word = marker_overlap("commission", "commission", freqs) + + assert word < symbol + + +def test_marker_frequencies_count_posts_not_occurrences(): + """A creator who repeats an emoji six times in one body has used it once as + far as identity goes.""" + counts = marker_frequencies(["\U0001F348 \U0001F348 \U0001F348 \U0001F348", "\U0001F348"]) + + assert counts["\U0001F348"] == 2 + + +def test_the_marker_gate_is_tighter_than_the_filename_gate(): + """Stated as a property because the two caps count different things and the + difference is deliberate: the filename cap counts a working name across a + piece's EXPORTS, the marker cap counts a public decoration across POSTS. A + marker tying an announcement to its drop lands on two posts — the two.""" + assert MAX_MARKER_FREQUENCY < MAX_TOKEN_FREQUENCY + + +def test_marker_overlap_cannot_be_called_without_the_frequencies(): + """An ungated call is the bug the signature exists to make impossible to + write by accident. A default would have kept it one keyword away.""" + with pytest.raises(TypeError): + marker_overlap("\U0001F348", "\U0001F348")