diff --git a/backend/app/services/post_association_service.py b/backend/app/services/post_association_service.py index ddab64f..737fdc8 100644 --- a/backend/app/services/post_association_service.py +++ b/backend/app/services/post_association_service.py @@ -29,27 +29,42 @@ additive, weighted, and no single one of its signals may reach the threshold: marker is in THIS artist's posts, because a habitual emoji is punctuation. IDENTITY evidence says two things are the same thing, and it gets its own -route (see `IDENTITY_FLOOR`): +route (see `IDENTITY_FLOOR`). Two signals, and the stronger one stands rather +than them being summed — saying "the same piece" twice is not more true: 4. **A shared working name.** The creator exports the teaser and the release from one file, and the internal name survives into both platforms untouched. Measured on the operator's artist: `ConnFront` ↔ `ConnFront`. This is the only signal that reaches a pair 23.8 hours apart, which proximity scores at 0.005. +5. **The drop contains the teaser's image.** Rare, and near-certain when it + happens. It is the one signal needing no cooperation from the creator: it + works on a teaser called `Screenshot 2026-08-13`, and on a creator whose + two platforms share no naming convention. ## The one deliberately NOT built -**Crop-to-source matching is HELD, on the plan's own instruction** — it is -real work with real false-positive risk, and it is only worth building once -the cheap signals are shown to be insufficient against the operator's actual -artists. Half of this creator's recent teasers are screenshots carrying no -working name at all, and those pairs are out of reach here; that, measured, is -what would justify it. +**Crop-to-source matching stays held, and now for a measured reason rather +than a cautious one.** -Note also that a naive whole-image SigLIP similarity is NOT that signal. A -cropped teaser and its full version are exactly the pair a whole-image -comparison handles worst, so adding one as a "bonus" would mostly add noise -while looking like progress. +It was deferred until the cheap signals could be shown insufficient. They can: +of artist 8's 27 teasers with a drop inside a day, 11 still go unlinked, and +five of those are screenshot teasers carrying no working name at all. + +So it was tried, on those exact pairs. Every teaser image was correlated +against every window of every nearby drop image at five scales, with the pairs +the working name independently confirms as ground truth and unrelated +same-artist posts a month away as a control. **It does not separate.** True +pairs score as low as 0.401 while the control reaches 0.605 — the two +distributions overlap, and no threshold divides them. + +The reason is the reason the naive version was rejected in the first place, +and it turns out to apply just as hard to the sophisticated one: one artist's +work is stylistically homogeneous, so any whole-image comparison between two +of their pieces is high whether or not it is the same piece. Signal 5 above is +what survived that experiment — it asks a narrower question ("is this the same +image") that the measurement shows is answerable, instead of a broader one +("is this a crop of that") that it shows is not. ## Creator identity comes free, so E4 is not actually a prerequisite @@ -81,12 +96,15 @@ from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from ..models import ImageRecord, ImportSettings, Post, PostAssociation +from ..utils.phash import hamming, hash_bits from ..utils.text import html_to_plain from .discord_grouping import DROP_GROUPER from .post_naming import ( IDENTITY_FLOOR, + MAX_TOKEN_POSTS, marker_frequencies, marker_overlap, + rarity, shared_identity, token_frequencies, working_name_tokens, @@ -143,6 +161,24 @@ MAX_RECENT_DROPS = 200 # are conclusive, 2 more propose, 2 fall short of both. AUTO_LINK_FLOOR = 1.0 +# When the drop simply CONTAINS the teaser's image — a pHash within this many +# of 256 bits. +# +# 32, the same number and unit `gallery_service._diversify_similar` already +# calls a near-duplicate. Measured on artist 8, comparing every teaser against +# every drop within a day: pairs the working name independently confirms score +# 0, 0 and 20, and the nearest unrelated same-artist pair in a 29-sample +# control scores **108**. A 76-bit gap, so the threshold is not finely tuned +# and does not need to be. +# +# `utils/phash` warns that the hash alone must not decide a MERGE, because +# variants of one piece collide at this distance. That warning does not invert +# here, it is the point: merging destroys a file, so a variant colliding with +# its original is a loss, while this is asking whether two POSTS are about the +# same piece — and a variant of the drop's image is exactly that. Nothing is +# deleted either way, so no pixel confirm is needed to accept. +DUPLICATE_MAX_DISTANCE = 32 + @dataclass(frozen=True) class _Corpus: @@ -159,6 +195,8 @@ class _Corpus: token_posts: Counter[str] text_by_post: dict[int, str] marker_posts: Counter[str] + hashes_by_post: dict[int, list[int]] + hash_posts: Counter[int] def proximity_signal(gap: timedelta, window: timedelta) -> float: @@ -195,6 +233,40 @@ def declared_signal(description: str | None) -> float: return 0.0 +def shared_image( + left: list[int], + right: list[int], + hash_posts: Counter[int], + *, + max_distance: int = DUPLICATE_MAX_DISTANCE, + max_frequency: int = MAX_TOKEN_POSTS, +) -> float: + """Strength in [0, 1] that the drop contains the teaser's own image. + + IDENTITY evidence, and the only one of the three that needs no cooperation + from the creator — it works on a teaser named `Screenshot 2026-08-13`, and + on a creator whose two platforms share no naming convention at all. Where + it fires it is close to certain; it is simply quiet most of the time, + because a teaser is usually a crop rather than a copy. + + Rarity-gated on POSTS like the other two: an image the creator puts on many + posts is a banner, not a piece. + """ + if not left or not right: + return 0.0 + best = None + for a in left: + for b in right: + d = hamming(a, b) + if d is None or d > max_distance: + continue + span = max(hash_posts.get(a, 1), hash_posts.get(b, 1), 1) + strength = rarity(span, max_frequency) + if best is None or strength > best: + best = strength + return round(best, 4) if best is not None else 0.0 + + def weighted_score(signals: dict) -> float: return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) @@ -213,14 +285,20 @@ class PostAssociationService: return self._corpora[artist_id] paths_by_post: dict[int, list[str]] = {} + hashes_by_post: dict[int, list[int]] = {} rows = await self.session.execute( - select(ImageRecord.primary_post_id, ImageRecord.path).where( + select( + ImageRecord.primary_post_id, ImageRecord.path, ImageRecord.phash + ).where( ImageRecord.artist_id == artist_id, ImageRecord.primary_post_id.is_not(None), ) ) - for post_id, path in rows: + for post_id, path, phash in rows: paths_by_post.setdefault(post_id, []).append(path) + bits = hash_bits(phash) + if bits is not None: + hashes_by_post.setdefault(post_id, []).append(bits) text_by_post: dict[int, str] = {} rows = await self.session.execute( @@ -243,6 +321,14 @@ class PostAssociationService: token_posts=token_frequencies(paths_by_post.values()), text_by_post=text_by_post, marker_posts=marker_frequencies(text_by_post.values()), + hashes_by_post=hashes_by_post, + # An image the creator puts on many posts — a banner, a watermark + # plate, a recurring title card — is a habit exactly as a character + # name is, and gets gated the same way. Counted on the EXACT hash, + # which is what a reused file produces. + hash_posts=Counter( + h for hs in hashes_by_post.values() for h in set(hs) + ), ) self._corpora[artist_id] = corpus return corpus @@ -325,17 +411,28 @@ class PostAssociationService: corpus = await self._corpus(announcement.artist_id) here = corpus.tokens_by_post.get(announcement.id, set()) here_text = corpus.text_by_post.get(announcement.id, "") + here_hashes = corpus.hashes_by_post.get(announcement.id, []) made = 0 scored: list[tuple[Post, float, dict, float]] = [] for group in await self._candidate_groups(announcement, window=window): if group.id in already: continue - identity, token = shared_identity( + named, token = shared_identity( here, corpus.tokens_by_post.get(group.id, set()), corpus.token_posts, ) + # The two identity signals answer the same question by different + # means, so the stronger one stands rather than them being summed: + # a name and a shared image both say "the same piece", and saying + # it twice is not more true. + copied = shared_image( + here_hashes, + corpus.hashes_by_post.get(group.id, []), + corpus.hash_posts, + ) + identity = max(named, copied) circumstantial = { "proximity": proximity_signal( _post_time(group) - _post_time(announcement), window, @@ -370,7 +467,9 @@ class PostAssociationService: if score < threshold: continue signals = {**circumstantial, "identity": identity} - if token: + if copied: + signals["identity_image"] = copied + if token and named >= copied: # Carried so the queue can say WHY. A review queue that cannot # explain itself is one the operator learns to click through. signals["identity_token"] = token diff --git a/backend/app/services/post_naming.py b/backend/app/services/post_naming.py index 9137cc2..e63aff4 100644 --- a/backend/app/services/post_naming.py +++ b/backend/app/services/post_naming.py @@ -214,10 +214,10 @@ def token_frequencies(posts: Iterable[Iterable[str]]) -> Counter[str]: return counts -def _rarity(freq: int, max_frequency: int) -> float: +def rarity(freq: int, max_frequency: int) -> float: """Rarity of one token within an artist's own corpus, in [0, 1]. - Shared by BOTH signals deliberately. They carried one formula each + Shared by EVERY rarity-gated signal deliberately. They carried one each until 2026-09-24, and the copies drifted: the filename signal grew a frequency gate and the marker signal never did, so a creator's habitual emoji scored the same 1.00 as a marker they had used twice. One @@ -262,7 +262,7 @@ def shared_identity( # The rarest shared token decides — one decisive token beats three vague # ones. token = min(shared, key=lambda t: (frequencies.get(t, 0), -len(t), t)) - strength = round(_rarity(max(frequencies.get(token, 1), 1), max_frequency), 4) + strength = round(rarity(max(frequencies.get(token, 1), 1), max_frequency), 4) # A token sitting exactly ON the cap decays to zero, and naming it anyway # would hand the review queue a reason that carries no weight — "matched on # loislanetb2", with nothing behind it. Measured: that token is on 6 of this @@ -372,7 +372,7 @@ def marker_overlap( if not shared: return 0.0 score = sum( - (1.0 if _SYMBOL.match(t) else 0.25) * _rarity(frequencies.get(t, 1), max_frequency) + (1.0 if _SYMBOL.match(t) else 0.25) * rarity(frequencies.get(t, 1), max_frequency) for t in shared ) return round(min(1.0, score), 4) diff --git a/backend/app/utils/phash.py b/backend/app/utils/phash.py index 4592b3a..66d127a 100644 --- a/backend/app/utils/phash.py +++ b/backend/app/utils/phash.py @@ -80,6 +80,33 @@ def _seek_first_frame(pil_image) -> None: pass +def hash_bits(hex_str: str | None) -> int | None: + """A stored pHash hex string as an integer, or None if it is missing or + unparseable. Fails CLOSED, like every other gate in this module. + + Parsed to an int rather than an imagehash object because the caller that + needs this compares one image against many: `int.bit_count()` on an XOR is + a machine instruction, where rebuilding a 16x16 boolean array per + comparison is not. + """ + if not hex_str: + return None + try: + return int(hex_str, 16) + except (TypeError, ValueError): + return None + + +def hamming(a: int | None, b: int | None) -> int | None: + """Bits differing between two parsed hashes, or None if either is absent. + + Out of 256 at HASH_SIZE 16. + """ + if a is None or b is None: + return None + return (a ^ b).bit_count() + + def compute_phash(pil_image) -> str | None: """Perceptual hash of an opened PIL image, as a hex string. None on any failure (videos/unreadable/non-image). diff --git a/tests/test_post_association.py b/tests/test_post_association.py index c7c6bed..ff9a890 100644 --- a/tests/test_post_association.py +++ b/tests/test_post_association.py @@ -7,6 +7,7 @@ of what follows pins refusals, and the central one is structural rather than behavioural: the threshold sits above every single signal weight, which is what makes "time proximity alone is never sufficient" arithmetic instead of a hope. """ +from collections import Counter from datetime import UTC, datetime, timedelta import pytest @@ -23,11 +24,13 @@ from backend.app.models import ( from backend.app.services.discord_grouping import DROP_GROUPER from backend.app.services.post_association_service import ( DECLARED_MENTION, + DUPLICATE_MAX_DISTANCE, WEIGHTS, PostAssociationService, declared_signal, proximity_signal, rescan, + shared_image, weighted_score, ) from backend.app.services.post_feed_service import PostFeedService @@ -131,13 +134,18 @@ async def _artist_with_channels(db, name: str): return artist, patreon, discord -async def _images(db, artist, post, ext, names): +async def _images(db, artist, post, ext, names, phashes=None): """Attach named files to a post. The NAME is the point — the working-name - signal reads it, so a test that cares about identity supplies one.""" + signal reads it, so a test that cares about identity supplies one. + + `phashes` aligns with `names`; a test that cares about the shared-image + signal supplies those instead (or as well). + """ for i, name in enumerate(names): db.add(ImageRecord( path=f"/images/{artist.id}/{ext}_{i}_{name}.jpg", sha256=f"{ext}{i}{name}".ljust(64, "0")[:64], + phash=(phashes or [None] * len(names))[i], size_bytes=10, mime="image/jpeg", width=10, height=10, origin="downloaded", primary_post_id=post.id, artist_id=artist.id, )) @@ -145,19 +153,19 @@ async def _images(db, artist, post, ext, names): async def _teaser(db, artist, source, *, at, body, ext="teaser", names=None, - title="New piece"): + title="New piece", phashes=None): post = Post( source_id=source.id, artist_id=artist.id, external_post_id=ext, post_date=at, post_title=title, description=body, ) db.add(post) await db.flush() - await _images(db, artist, post, ext, names or [ext]) + await _images(db, artist, post, ext, names or [ext], phashes) return post async def _drop(db, artist, source, *, at, ext="fc-drop:1", body=None, - names=()): + names=(), phashes=None): post = Post( source_id=source.id, artist_id=artist.id, external_post_id=ext, post_date=at, description=body, synthesized_by=DROP_GROUPER, @@ -166,7 +174,7 @@ async def _drop(db, artist, source, *, at, ext="fc-drop:1", body=None, db.add(post) await db.flush() if names: - await _images(db, artist, post, ext.replace(":", "-"), names) + await _images(db, artist, post, ext.replace(":", "-"), names, phashes) return post @@ -831,3 +839,114 @@ async def test_a_drop_another_post_already_claims_is_never_taken(db): ) )).scalars().all() assert [r.status for r in rows] == ["pending"] + + +# --- when the drop just contains the teaser's image ------------------------- +# +# Measured on artist 8, every teaser against every drop within a day: pairs the +# working name independently confirms score 0, 0 and 20 bits of 256, and the +# nearest unrelated same-artist pair in a 29-sample control scores 108. The +# threshold sits at 32 — the same number gallery_service already calls a +# near-duplicate — inside a 76-bit gap. + +_PIECE = "a5" * 32 # the image +_REENCODED = "a5" * 31 + "a4" # the same image, one bit different +_UNRELATED = "5a" * 32 # 256 bits away — every bit differs + + +@pytest.mark.asyncio +async def test_the_drop_carrying_the_teasers_own_image_links_it(db): + """The one signal that needs no cooperation from the creator. No shared + name, nothing said about Discord, 20 hours apart — and the drop is + carrying the same picture.""" + artist, patreon, discord = await _artist_with_channels(db, "dupartist") + now = datetime.now(UTC) + teaser = await _teaser( + db, artist, patreon, at=now - timedelta(hours=20), body="a preview", + ext="t0", names=["Alpha"], phashes=[_PIECE], + ) + await _drop(db, artist, discord, at=now, names=["Beta"], + phashes=[_REENCODED]) + await db.commit() + + made = await PostAssociationService(db).match_post( + teaser.id, threshold=DEFAULT_THRESHOLD, window_hours=WINDOW, + auto_link=True, + ) + await db.commit() + + assert made == (1, 1) + assoc = (await db.execute(select(PostAssociation))).scalar_one() + assert assoc.signals["identity_image"] == 1.0 + assert "identity_token" not in assoc.signals, ( + "the names share nothing — claiming one would be a false reason" + ) + + +@pytest.mark.asyncio +async def test_a_different_picture_links_nothing(db): + """The negative the threshold exists for. 256 bits apart is two different + images, whatever else the posts have in common.""" + artist, patreon, discord = await _artist_with_channels(db, "diffartist") + now = datetime.now(UTC) + teaser = await _teaser( + db, artist, patreon, at=now - timedelta(hours=20), body="a preview", + ext="t0", names=["Alpha"], phashes=[_PIECE], + ) + await _drop(db, artist, discord, at=now, names=["Beta"], + phashes=[_UNRELATED]) + await db.commit() + + made = await PostAssociationService(db).match_post( + teaser.id, threshold=DEFAULT_THRESHOLD, window_hours=WINDOW, + auto_link=True, + ) + await db.commit() + + assert made == (0, 0) + + +@pytest.mark.asyncio +async def test_an_image_the_creator_reuses_everywhere_links_nothing(db): + """The same guard the other two signals have, on the third. A banner, a + watermark plate or a recurring title card is a habit, not a piece — and it + would otherwise link every post carrying it to every drop carrying it.""" + artist, patreon, discord = await _artist_with_channels(db, "bannerartist") + now = datetime.now(UTC) + teaser = await _teaser( + db, artist, patreon, at=now - timedelta(hours=20), body="a preview", + ext="t0", names=["Alpha"], phashes=[_PIECE], + ) + await _drop(db, artist, discord, at=now, names=["Beta"], + phashes=[_REENCODED]) + for i in range(7): + await _teaser( + db, artist, patreon, at=now - timedelta(days=30 + i), body="older", + ext=f"other{i}", names=[f"Gamma{i}"], phashes=[_PIECE], + ) + await db.commit() + + svc = PostAssociationService(db) + made = await svc.match_post( + teaser.id, threshold=DEFAULT_THRESHOLD, window_hours=WINDOW, + auto_link=True, + ) + await db.commit() + + assert made == (0, 0) + corpus = await svc._corpus(artist.id) + assert corpus.hash_posts[int(_PIECE, 16)] == 8 + + +def test_a_missing_hash_is_not_a_match(): + """Fails CLOSED, like every gate in utils/phash. An image whose pHash was + never computed must read as "no evidence", never as "identical to the + other thing that also has none".""" + assert shared_image([], [], Counter()) == 0.0 + + +def test_the_duplicate_threshold_sits_inside_the_measured_gap(): + """Stated as a property so the number cannot drift out of the gap that + justifies it: 20 bits was the widest true pair, 108 the nearest unrelated + one.""" + assert 20 < DUPLICATE_MAX_DISTANCE < 108