backend/app/services/post_naming.py

@@ -0,0 +1,319 @@
+"""The creator's own working name for a piece, recovered from a filename.
+
+Milestone 388. Pure functions, no DB and no ML — the whole point is that this
+signal is free.
+
+## What this is for
+
+Two of the operator's artists post a censored or cropped teaser on Patreon and
+the real release in their Discord. Matching those by IMAGE is the pair a
+whole-image comparison handles worst: the teaser is a crop with a censor bar,
+which is exactly the local edit that moves a perceptual hash and blurs a
+semantic embedding.
+
+But the creator names both exports after the same internal working title, and
+that name survives into both platforms untouched. Measured on the live instance
+2026-09-24, artist 8:
+
+    01_((0-k           <->  0-k_base          (1.3h apart)
+    01_680LC           <->  680LC_Border      (21.0h apart)
+    01_cnni18x         <->  cnni18x           (21.5h apart)
+
+Three pairs, no false positives, and **two of them are 21 hours apart** — far
+enough that time proximity scores them ~0.10 and could never propose them. The
+naming signal is orthogonal to the timing one: each finds pairs the other
+cannot, which is why both are kept rather than one being tuned to cover both.
+
+## Why a filename and not a perceptual hash
+
+A shared working-name token is IDENTITY evidence — `680lc` appearing on both
+platforms is not a coincidence. Proximity is CIRCUMSTANTIAL: it says two things
+happened near each other, never that they are the same thing. The distinction
+drives the weighting in `post_association_service`, and it is why a rare enough
+token is allowed to carry a proposal on its own while no amount of circumstance
+is.
+
+## The one false-positive class found, and why the fix is shaped this way
+
+A first pass matched `01_Screenshot 2026-08-13 000004` to
+`Screenshot_2026-08-13_032144` on the token `2026-08-13`, twice.
+
+A screenshot filename is a camera artifact. It carries no working name, and the
+date inside it collides across platforms on the same day BY CONSTRUCTION — the
+teaser and the release are posted the same day, so their screenshot names
+always share a date token. That is a signal that fires precisely when it is
+least informative.
+
+So a filename with no working name contributes NOTHING, rather than the
+plausible-looking date match it could be squeezed for. Re-run with that rule:
+the same three true pairs, zero false. Half of this creator's recent teasers
+are screenshots, and those pairs are simply out of this signal's reach — which
+is where crop-to-source matching earns its cost, and nowhere else.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from collections.abc import Iterable
+from pathlib import PurePosixPath
+
+# A screenshot name, on either platform. Patreon's importer writes
+# `01_Screenshot 2026-08-13 000004`; gallery-dl's Discord naming writes
+# `Screenshot_2026-09-22_003651`. Matched after the index/message prefixes are
+# stripped, so both shapes reach this as a bare `Screenshot ...`.
+# NOT `\b` after "shot": `\b` needs a word/non-word transition and `_` is a
+# WORD character, so `Screenshot_2026-08-13_032144` — gallery-dl's Discord
+# spelling — sailed straight past the guard while the space-separated Patreon
+# spelling was caught. Found by running this against the live library rather
+# than by reading it. Assert the next character is not a letter instead.
+_SCREENSHOT = re.compile(r"^screen[ _-]?shot(?![a-z])", re.I)
+
+# The importer's per-post media index: `01_`, `02_`. Not part of any name.
+_MEDIA_INDEX = re.compile(r"^\d{1,3}_")
+
+# gallery-dl's Discord filename pattern (#3999):
+# `{date:%Y%m%d}_{message_id}_{num:>02}_{filename}`.
+_DISCORD_PREFIX = re.compile(r"^\d{8}_\d{6,}_\d{1,3}_")
+
+# The importer's content-hash suffix, `__<10 hex>`, sometimes doubled on files
+# that went through an older import era.
+_HASH_SUFFIX = re.compile(r"(?:__[0-9a-f]{10})+$")
+
+# Generic export decorations. Stripped as SUFFIXES so the stem survives:
+# `cnni18x_wip3` and `cnni18x` must yield the same token, or a work-in-progress
+# would never match the piece it became.
+_DECORATION = re.compile(
+    r"(?:[_-]?(?:wip|base|final|alt|alts|edit|edits|border|clean|raw|hd|full|"
+    r"censored|uncensored|nsfw|sfw|ver|v)\d*)+$",
+    re.I,
+)
+
+# Tokens that carry no identity even when they survive the rules above.
+_STOPWORDS = frozenset({
+    "img", "image", "untitled", "new", "test", "page", "final", "copy",
+    "post", "media", "file", "avatar", "cover", "banner", "icon", "splash",
+    # Literally the string "None": issue #3999's Discord naming rendered
+    # `{user[name]}` as it for ~1,600 files, so it is the single most common
+    # "name" in the library and identifies nothing.
+    "none",
+})
+
+# A bare year, or a date fragment. These are what made the screenshot collision
+# look like a match, and they are worthless as identity even in a real name.
+_YEAR = re.compile(r"^(?:19|20)\d{2}$")
+_ALL_DIGITS = re.compile(r"^\d+$")
+
+MIN_TOKEN_LEN = 3
+
+# A token shared by more than this many of ONE ARTIST's images is a habit, not
+# an identity — a character name, a series tag, a recurring export preset. The
+# bar is deliberately low: a working name identifies one piece, so it should
+# appear on that piece's handful of exports (base, wips, the teaser crop) and
+# nowhere else. Raising this trades false positives for reach, which is the
+# wrong direction here — a wrong link asserts two different pieces are one.
+MAX_TOKEN_FREQUENCY = 6
+
+
+def _strip_prefixes(stem: str) -> str:
+    """Remove the framing each platform's importer adds around the real name."""
+    stem = _DISCORD_PREFIX.sub("", stem)
+    stem = _MEDIA_INDEX.sub("", stem)
+    return _HASH_SUFFIX.sub("", stem)
+
+
+def working_name_tokens(path: str) -> set[str]:
+    """The identity-bearing tokens in one image's filename.
+
+    Returns an EMPTY set for a name that carries no working title — a
+    screenshot, a bare number, a stopword. Empty means "no evidence", which the
+    caller must treat as silence rather than as a weak match; see the module
+    docstring for the false positive that rule exists for.
+    """
+    stem = _strip_prefixes(PurePosixPath(path).stem)
+    if _SCREENSHOT.match(stem.strip()):
+        return set()
+
+    out: set[str] = set()
+    # Hyphens are kept INSIDE tokens — `0-k` is a real working name on the live
+    # instance, and splitting on hyphen would reduce it to a single character
+    # and then discard it for being too short.
+    for raw in re.split(r"[^0-9A-Za-z-]+", stem.lower()):
+        tok = _DECORATION.sub("", raw).strip("-")
+        if len(tok) < MIN_TOKEN_LEN:
+            continue
+        if tok in _STOPWORDS or _YEAR.match(tok) or _ALL_DIGITS.match(tok):
+            continue
+        out.add(tok)
+    return out
+
+
+def token_frequencies(paths: Iterable[str]) -> Counter[str]:
+    """How often each working-name token appears across one artist's images.
+
+    Scoped to the ARTIST, not the library: a working name belongs to the person
+    who chose it, and the same string can be one creator's piece and another's
+    boilerplate. Built once per artist per sweep, not per candidate pair.
+    """
+    counts: Counter[str] = Counter()
+    for p in paths:
+        counts.update(working_name_tokens(p))
+    return counts
+
+
+def _rarity(freq: int, max_frequency: int) -> float:
+    """Rarity of one token within an artist's own corpus, in [0, 1].
+
+    Shared by BOTH signals deliberately. They carried one formula each
+    until 2026-09-24, and the copies drifted: the filename signal grew a
+    frequency gate and the marker signal never did, so a creator's habitual
+    emoji scored the same 1.00 as a marker they had used twice. One
+    definition cannot drift from itself.
+
+    Full strength at 2 rather than 1: a genuine match means the token is on
+    at least two things, so demanding uniqueness would reject every real
+    pair. Decays to zero AT the cap rather than falling off it, so nothing
+    sits on a cliff edge.
+    """
+    if freq <= 2:
+        return 1.0
+    if freq >= max_frequency:
+        return 0.0
+    return (max_frequency - freq) / (max_frequency - 2)
+
+
+def shared_identity(
+    left: Iterable[str],
+    right: Iterable[str],
+    frequencies: Counter[str],
+    *,
+    max_frequency: int = MAX_TOKEN_FREQUENCY,
+) -> tuple[float, str | None]:
+    """Strength in [0, 1] that two sets of filenames name the SAME piece.
+
+    Returns `(strength, token)` — the token is carried back so the proposal can
+    say WHY it was made. A review queue that cannot explain itself is one the
+    operator learns to click through without reading.
+
+    Strength is a function of the winning token's rarity within the artist's
+    own library, not of how many tokens matched. One decisive token beats three
+    vague ones, and a token that appears on forty of this artist's images is a
+    habit rather than an identity however exactly it matches.
+    """
+    shared = {t for t in set(left) & set(right) if frequencies.get(t, 0) <= max_frequency}
+    if not shared:
+        return 0.0, None
+
+    # The rarest shared token decides — one decisive token beats three vague
+    # ones. `frequencies` counts IMAGES here, and a real working name lands on
+    # a few of them: the base, its wips, the teaser crop.
+    token = min(shared, key=lambda t: (frequencies.get(t, 0), -len(t), t))
+    strength = round(_rarity(max(frequencies.get(token, 1), 1), max_frequency), 4)
+    # A token sitting exactly ON the cap decays to zero, and naming it anyway
+    # would hand the review queue a reason that carries no weight — "matched on
+    # loislanetb2", with nothing behind it. Measured: that token is on 6 of this
+    # artist's images. Report a token only when it is doing work.
+    return (strength, token) if strength > 0 else (0.0, None)
+
+
+# --- the body/title signal ---------------------------------------------------
+#
+# The same idea applied to TEXT. The operator's example pair carries `🍈🍈` in
+# the Patreon title and `@everyone 🍈 🍈` in the Discord message — a marker the
+# creator uses to tie the two together, which no vocabulary list would predict.
+#
+# Rarity-gated, exactly as the filename signal is, and the gate is here because
+# the first pass did NOT have one. Measured on artist 8, 300 posts:
+#
+#     💦 11 posts (4%)   🫴 6   🌰 5   🍗 5   🫣 4
+#
+# 💦 is punctuation for this creator — about one post in twenty-five. Ungated it
+# scored a full 1.00 and was the DECIDING term in a proposal that proximity
+# alone (0.441) could not carry. A habitual marker riding along with proximity
+# is just proximity wearing a hat, which is the exact failure the matcher's
+# threshold sits above 0.55 to prevent. The operator's 🍈🍈 is the opposite
+# case: two posts, and they are the pair itself.
+
+_WORD = re.compile(r"[0-9A-Za-z]{3,}")
+# Anything outside the Basic Multilingual Plane's text ranges: emoji, symbols,
+# kaomoji parts. These are the tokens creators actually use as markers, and
+# they are rare enough in prose to be evidence on their own.
+# U+1F000-1FAFF is the emoji planes; U+2190-2BFF covers arrows, dingbats and
+# the miscellaneous-symbol blocks, which already contains U+2600-27BF.
+_SYMBOL = re.compile(r"[\U0001F000-\U0001FAFF\u2190-\u2BFF]")
+
+_COMMON_TEXT = frozenset({
+    "the", "and", "for", "you", "new", "out", "now", "this", "that", "with",
+    "everyone", "here", "post", "all", "art", "one", "get", "has", "are",
+})
+
+# A marker in more than this many of ONE ARTIST's posts is a signature, not a
+# tie-back. Tighter than MAX_TOKEN_FREQUENCY because the units differ and so
+# does the evidence: that one counts a working name across a piece's handful of
+# EXPORTS, where this counts a public decoration across POSTS. A marker that
+# ties an announcement to its drop lands on two posts — the two.
+MAX_MARKER_FREQUENCY = 4
+
+
+def text_markers(text: str | None) -> set[str]:
+    """Distinctive tokens in a post body or title: symbols, and rare-ish words.
+
+    Symbols count individually rather than as a run, so `🍈🍈` and `🍈 🍈` —
+    which is how the same marker appears on the two platforms — reduce to the
+    same token. Spacing is a platform's rendering, not the creator's intent.
+    """
+    if not text:
+        return set()
+    out = {m.group(0) for m in _SYMBOL.finditer(text)}
+    out |= {
+        w.lower() for w in _WORD.findall(text)
+        if w.lower() not in _COMMON_TEXT and not _YEAR.match(w)
+    }
+    return out
+
+
+def marker_frequencies(texts: Iterable[str | None]) -> Counter[str]:
+    """How many of ONE ARTIST's posts each marker appears in.
+
+    Per POST, not per occurrence: a creator who repeats an emoji six times in
+    one body has used it once as far as identity goes. Scoped to the artist for
+    the same reason `token_frequencies` is — a marker is a personal habit, and
+    one creator's signature is another's whole vocabulary.
+    """
+    counts: Counter[str] = Counter()
+    for t in texts:
+        counts.update(text_markers(t))
+    return counts
+
+
+def marker_overlap(
+    left: str | None,
+    right: str | None,
+    frequencies: Counter[str],
+    *,
+    max_frequency: int = MAX_MARKER_FREQUENCY,
+) -> float:
+    """Strength in [0, 1] that two texts share a DELIBERATE marker.
+
+    `frequencies` is required rather than defaulted to "no gate". An ungated
+    call is the bug this signature exists to make impossible to write by
+    accident, and a default would have kept it one keyword away.
+
+    Symbols weigh full and words a quarter, because prose shares words by
+    accident: a creator who writes "commission" in both posts on a Tuesday has
+    told us nothing that the timestamps did not already say.
+
+    There is no divisor. An earlier pass halved the total so that a long body
+    could not out-vote a short one, which the rarity gate now does properly —
+    and halving meant the operator's own 🍈🍈 pair, a marker on exactly two
+    posts, could reach only 0.5. One marker the creator uses nowhere else is
+    the whole signal, not half of it.
+    """
+    shared = text_markers(left) & text_markers(right)
+    if not shared:
+        return 0.0
+    score = sum(
+        (1.0 if _SYMBOL.match(t) else 0.25) * _rarity(frequencies.get(t, 1), max_frequency)
+        for t in shared
+    )
+    return round(min(1.0, score), 4)

tests/test_post_naming.py

@@ -0,0 +1,233 @@
+"""Milestone 388: the creator's own working name, recovered from a filename.
+
+Two of the operator's artists post a censored or cropped teaser on Patreon and
+the real release in their Discord. That pair is the one a whole-image
+comparison handles WORST — a crop with a censor bar is exactly the local edit
+that moves a perceptual hash and blurs a semantic embedding — and it is the
+pair the creator names identically on both platforms.
+
+Everything here was calibrated against the live library (artist 8, 520 images,
+409 distinct tokens) rather than invented, so the cases carry their measured
+numbers. The property the whole module serves is that a WRONG link is worse
+than no link: no link leaves the operator where they already were, a wrong one
+tells them two different pieces are the same piece.
+"""
+from collections import Counter
+
+import pytest
+
+from backend.app.services.post_naming import (
+    MAX_MARKER_FREQUENCY,
+    MAX_TOKEN_FREQUENCY,
+    marker_frequencies,
+    marker_overlap,
+    shared_identity,
+    text_markers,
+    token_frequencies,
+    working_name_tokens,
+)
+
+# --- what survives each platform's framing -----------------------------------
+#
+# Three naming eras reach this (#4002), and a token has to come out the same
+# from all of them or the signal only works on whichever era it was written
+# against.
+
+
+@pytest.mark.parametrize(
+    "path, expected, era",
+    [
+        (
+            "/images/yellowroom/patreon/2026-09-23_170392790_Anya/01_((0-k.jpg",
+            "0-k",
+            "current Patreon: a per-post media index",
+        ),
+        (
+            "20260923_1552424119805673604_01_0-k_base.jpg",
+            "0-k",
+            "Discord: gallery-dl's date_messageid_num_ prefix (#3999)",
+        ),
+        (
+            "85317841_media_212565911_0071 NoHeart__c3118a69f3__c3118a69f3.jpg",
+            "noheart",
+            "legacy: flat at the artist root, post id + media id + doubled hash",
+        ),
+    ],
+)
+def test_one_working_name_survives_every_naming_era(path, expected, era):
+    assert expected in working_name_tokens(path), era
+
+
+def test_a_hyphen_stays_inside_a_token():
+    """`0-k` is a real working name on the live instance — it is the operator's
+    own example pair. Splitting on hyphen reduces it to two single characters
+    and then discards both for being too short, which silently loses the one
+    case this was built to catch."""
+    assert working_name_tokens("01_((0-k.jpg") == {"0-k"}
+
+
+def test_a_work_in_progress_matches_the_piece_it_became():
+    """Measured: `cnni6600 wip1` (Patreon) and `cnni6600_Base` (Discord) are
+    one piece. A decoration is a stage, not a name."""
+    assert working_name_tokens("01_cnni6600 wip1.png") == working_name_tokens(
+        "20250514_1372285581945471087_01_cnni6600_Base.jpg"
+    )
+
+
+def test_the_literal_string_none_is_not_a_name():
+    """#3999 rendered `{user[name]}` as "None" for ~1,600 files, which made it
+    the single most common "name" in the library and an identity for nothing."""
+    assert working_name_tokens("20180508_None_DCthingwhat__0123456789.jpg") == {
+        "dcthingwhat"
+    }
+
+
+# --- the false positive that shaped the design -------------------------------
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "01_Screenshot 2026-08-13 000004.png",       # Patreon's spelling
+        "20260813_1234567890123_01_Screenshot_2026-08-13_032144.png",  # Discord's
+    ],
+)
+def test_a_screenshot_contributes_nothing_at_all(path):
+    """A first pass matched these two to each other on the token `2026-08-13`.
+
+    A screenshot filename is a camera artifact carrying no working name, and
+    its date collides across platforms BY CONSTRUCTION: the teaser and the
+    release go out the same day, so their screenshot names always share one.
+    That is a signal firing precisely when it is least informative.
+
+    Both spellings matter. `\\b` after "shot" was the first guard and it caught
+    only the Patreon one, because `_` is a WORD character so
+    `Screenshot_2026-08-13` never presented a word boundary there. Found by
+    running against the live library, not by reading the regex.
+    """
+    assert working_name_tokens(path) == set()
+
+
+# --- rarity, which both signals now share ------------------------------------
+
+
+def test_a_token_on_two_images_is_full_strength():
+    """Two is the floor rather than one: a genuine match means the token is on
+    at least two files, so demanding uniqueness rejects every real pair."""
+    freqs = Counter({"connfront": 2})
+
+    assert shared_identity({"connfront"}, {"connfront"}, freqs) == (1.0, "connfront")
+
+
+def test_a_character_name_can_never_link_two_posts():
+    """THE false-positive guard, and the reason the cap is set where it is.
+    Measured on artist 8: `anya` is on 8 images, `riju` 9, `undyne` 8, `bea` 16.
+    Ungated, every Anya post would match every Anya drop."""
+    freqs = Counter({"anya": 8})
+
+    assert shared_identity({"anya"}, {"anya"}, freqs) == (0.0, None)
+
+
+def test_a_token_at_the_cap_names_nothing():
+    """It decays to zero, and reporting it anyway would hand the review queue a
+    reason with no weight behind it — "matched on loislanetb2", with nothing
+    there. A token is named only while it is doing work."""
+    freqs = Counter({"tok": MAX_TOKEN_FREQUENCY})
+
+    assert shared_identity({"tok"}, {"tok"}, freqs) == (0.0, None)
+
+
+def test_the_rarest_shared_token_decides_not_the_count_of_them():
+    """One decisive token beats three vague ones. A pair sharing a piece name
+    AND two habits is evidence of the piece name."""
+    freqs = Counter({"rare": 2, "goo": 9, "shiny": 7})
+
+    strength, token = shared_identity(
+        {"rare", "goo", "shiny"}, {"rare", "goo", "shiny"}, freqs
+    )
+
+    assert (strength, token) == (1.0, "rare")
+
+
+def test_frequencies_are_counted_per_artist_not_per_library():
+    """A working name belongs to the person who chose it; the same string is
+    one creator's piece and another's boilerplate."""
+    counts = token_frequencies(
+        ["01_ConnFront.jpg", "20230222_1078078245695664148_01_ConnFront.jpg"]
+    )
+
+    assert counts["connfront"] == 2
+
+
+# --- the text marker, gated the same way -------------------------------------
+
+
+def test_the_same_marker_reads_the_same_through_both_platforms():
+    """The operator's pair carries `\U0001F348\U0001F348` in the Patreon title and
+    `@everyone \U0001F348 \U0001F348` in the Discord message. Spacing is a platform's
+    rendering, not the creator's intent."""
+    assert "\U0001F348" in text_markers("Anya -- \U0001F348\U0001F348")
+    assert "\U0001F348" in text_markers("@everyone \U0001F348 \U0001F348")
+
+
+def test_one_marker_the_creator_uses_nowhere_else_is_the_whole_signal():
+    """Measured: `\U0001F317` is on exactly two of this artist's 300 posts, and they
+    are the pair. An earlier pass halved every total so a long body could not
+    out-vote a short one — which the rarity gate now does properly, and which
+    meant this case could reach only 0.5."""
+    freqs = Counter({"\U0001F317": 2})
+
+    assert marker_overlap("\U0001F317", "\U0001F317 drop", freqs) == 1.0
+
+
+def test_a_marker_the_creator_uses_habitually_is_worth_nothing():
+    """The bug this gate exists for. Measured on artist 8: \U0001F4A6 is in 13 of 300
+    posts — punctuation, about one post in twenty-five. Ungated it scored a
+    full 1.00 and was the DECIDING term in a proposal that proximity alone
+    could not carry, which is proximity wearing a hat."""
+    freqs = Counter({"\U0001F4A6": 13})
+
+    assert marker_overlap("Drizzle \U0001F4A6", "\U0001F4A6", freqs) == 0.0
+
+
+def test_a_shared_ordinary_word_is_worth_nothing():
+    """Also measured, also a real proposal: the shared "marker" between a
+    teaser and a drop was the word `like`, which is in 43 of this artist's
+    posts."""
+    freqs = Counter({"like": 43})
+
+    assert marker_overlap("like this", "like that", freqs) == 0.0
+
+
+def test_a_symbol_outweighs_a_word_at_equal_rarity():
+    """Prose shares words by accident; a creator who writes "commission" in
+    both posts on a Tuesday has said nothing the timestamps did not."""
+    freqs = Counter({"\U0001F317": 2, "commission": 2})
+
+    symbol = marker_overlap("\U0001F317", "\U0001F317", freqs)
+    word = marker_overlap("commission", "commission", freqs)
+
+    assert word < symbol
+
+
+def test_marker_frequencies_count_posts_not_occurrences():
+    """A creator who repeats an emoji six times in one body has used it once as
+    far as identity goes."""
+    counts = marker_frequencies(["\U0001F348 \U0001F348 \U0001F348 \U0001F348", "\U0001F348"])
+
+    assert counts["\U0001F348"] == 2
+
+
+def test_the_marker_gate_is_tighter_than_the_filename_gate():
+    """Stated as a property because the two caps count different things and the
+    difference is deliberate: the filename cap counts a working name across a
+    piece's EXPORTS, the marker cap counts a public decoration across POSTS. A
+    marker tying an announcement to its drop lands on two posts — the two."""
+    assert MAX_MARKER_FREQUENCY < MAX_TOKEN_FREQUENCY
+
+
+def test_marker_overlap_cannot_be_called_without_the_frequencies():
+    """An ungated call is the bug the signature exists to make impossible to
+    write by accident. A default would have kept it one keyword away."""
+    with pytest.raises(TypeError):
+        marker_overlap("\U0001F348", "\U0001F348")