diff --git a/.env.example b/.env.example index fe2cb12..a000892 100644 --- a/.env.example +++ b/.env.example @@ -93,7 +93,7 @@ DB_NAME=fabledcurator # # FabledCurator has no login, no accounts and no permission model. Anything # that can reach PORT is an administrator and can read the platform session -# cookies the app stores for Patreon, SubscribeStar and Pixiv. +# cookies the app stores for Patreon and SubscribeStar. # # Bind it to a trusted network. See "Before you expose it" in README.md and # the deployment posture section of SECURITY.md. diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index afe39e2..56248fd 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -172,10 +172,11 @@ jobs: mkdir -p /tmp/versions_held mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true - # Printed rather than uploaded: ci-requirements.md records that this - # runner cannot do actions/upload-artifact@v4+, and the repo dropped - # the action entirely in 2026-05, so the job log is the retrieval - # channel actually proven here. + # Printed rather than uploaded: the repo dropped actions/upload-artifact + # in 2026-05, when the runner could not run v4+, and the job log is the + # retrieval channel this job has proven. (gitea/runner 3.x runs stock + # upload-artifact now — Scribe snippet #2271 — so an artifact is an + # option if the log ever stops being enough.) # # base64, not the raw file. A plain `cat` of the ~33KB candidate was # TRUNCATED MID-LINE by the runner on run 4964 — it stopped inside diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index b37e0ba..fdfa770 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -445,11 +445,12 @@ jobs: trap - EXIT echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release" - # No actions/upload-artifact step: Forgejo Actions (and our - # act_runner) doesn't support upload-artifact@v4+ (GHES limitation - # surfaced 2026-05-26). Instead build-web reads the signed XPI - # straight from the ext- Forgejo release we just uploaded - # to. Same source of truth; no double-store. + # No actions/upload-artifact step: build-web reads the signed XPI + # straight from the ext- Forgejo release we just uploaded to. + # Same source of truth; no double-store. The step was dropped 2026-05-26 + # because act_runner could not run upload-artifact@v4+; gitea/runner 3.x + # can (Scribe snippet #2271), but the release asset stays the better + # channel for a file build-web needs on every run. build-web: # Consumed by smoke-web's job-level `if:`. It cannot read `env` — the env diff --git a/.gitignore b/.gitignore index c726541..ae5264c 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,22 @@ alembic/versions/__pycache__/ *.sqlite *.sqlite-journal .superpowers/ + +# Raw platform captures (milestone 387 C0 and successors). These are real +# authenticated API responses taken from the operator's own account, so they +# carry account data — creator lists, pledge amounts, and (in Patreon's case) +# the account email inside the `card` resources. They are kept locally because +# re-capturing means re-authenticating by hand, and they are the ground truth a +# characterization gets re-checked against. +# +# The whole directory is ignored, not one filename, so a future capture is +# covered by this rule instead of needing a new line somebody has to remember. +# +# SANITIZED fixtures derived from these DO belong in git — put them somewhere +# else (tests/fixtures/, not here), with the account data stripped. +# Ignore the CONTENTS, not the directory: git does not descend into an +# excluded directory, so a negation for a file inside one never takes effect. +# Writing it this way lets README.md be committed while everything else here +# stays out. +tests/fixtures/captures/* +!tests/fixtures/captures/README.md diff --git a/README.md b/README.md index 2770fd4..8c94e40 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + # FabledCurator @@ -11,11 +13,12 @@ tags it, and gives you something better than a folder full of images to look through afterwards. - **Gallery and browsing.** Images, videos and multi-page works, organised by - artist, tag, post and series. A Showcase front page, a filterable gallery, a - similarity-driven Explore view, and a page-turning reader for series. -- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Pixiv and - anything `gallery-dl` supports, on a schedule. Handles paywalled posts using - your own logged-in session. + artist, tag, post and series. A newest-first feed of what just arrived as the + front page, a random Showcase, a filterable gallery, a similarity-driven + Explore view, and a page-turning reader for series. +- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Discord and + HentaiFoundry, on a schedule. Handles paywalled posts using your own + logged-in session. - **ML tagging.** Runs image models in-container to suggest tags, group characters, find near-duplicates and power similarity search. Suggestions are reviewable — it proposes, you confirm, and it learns which proposals you keep @@ -36,8 +39,8 @@ is no config file to edit beyond a handful of bootstrap environment variables. permission model. Anything that can reach the port is an administrator. That matters more here than it would in most self-hosted apps, because of what -this one stores: **live platform session cookies for Patreon, SubscribeStar and -Pixiv** — accounts that usually have a payment method attached. Whoever reaches +this one stores: **live platform session cookies for Patreon and +SubscribeStar** — accounts that usually have a payment method attached. Whoever reaches the port can read them, alongside your entire library. So: diff --git a/SECURITY.md b/SECURITY.md index f59adcf..bad7648 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -26,7 +26,7 @@ FabledCurator is self-hosted and holds things worth stating plainly, because they shape what counts as a serious bug here: - **Platform credentials.** The app captures and stores session cookies for - third-party subscription sites (Patreon, SubscribeStar, Pixiv) so it can + third-party subscription sites (Patreon, SubscribeStar) so it can download on the operator's behalf. These are live credentials for accounts that usually carry a payment method. Anything that discloses them, decrypts them, or lets one user of a shared instance read another's is high severity. @@ -56,7 +56,7 @@ reverse proxy. It also does not authenticate anyone — see above. These are documented design decisions, not oversights. Putting this on the public internet, with or without TLS, hands whoever finds -it your Patreon, SubscribeStar and Pixiv sessions. A reverse proxy that adds +it your Patreon and SubscribeStar sessions. A reverse proxy that adds TLS but not an authentication layer does not change that. Reports that reduce to "the application is served over HTTP", "there is no diff --git a/alembic/versions/0091_platform_membership.py b/alembic/versions/0091_platform_membership.py new file mode 100644 index 0000000..899f6e4 --- /dev/null +++ b/alembic/versions/0091_platform_membership.py @@ -0,0 +1,81 @@ +"""platform_membership — the learned roster of what the account actually pays for. + +Milestone 387, phase C. FC knows which creators it was told to follow and +nothing about which ones the operator is subscribed to; this table is the +memory that makes the drift in both directions observable. See the model +docstring for why the roster is learned rather than looked up live, and why +`status` holds the platform's own word rather than a normalised FC value. + +## Nothing populates this yet, on purpose + +The sweep that fills it (C3) depends on a client seam (C2) that depends on +characterising Patreon's real membership response from a captured sample (C0), +which needs the operator's authenticated browser session. The table's SHAPE +does not wait on that: it is deliberately free-form where C0's findings would +otherwise dictate a column — `status` is an unconstrained String and `details` +keeps the raw payload — so no capture can invalidate what is created here. + +An empty table is the correct intermediate state. It is not dead code: C5 reads +it to explain a tier-limited source, and C4 reads it to reconcile. + +Revision ID: 0091 +Revises: 0090 +Create Date: 2026-09-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0091" +down_revision: Union[str, None] = "0090" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "platform_membership", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + # Text, not a bounded String: an opaque upstream identifier we do not + # mint, and guessing a ceiling for one is how a walk dies on a silent + # truncation. + sa.Column("external_campaign_id", sa.Text(), nullable=False), + sa.Column("display_name", sa.Text(), nullable=True), + sa.Column("url", sa.Text(), nullable=True), + # No CHECK, deliberately (rule 36 considered and declined): the + # vocabulary is each platform's own and is not ours to fix before C0 + # has characterised even one of them. The service owns the whitelist. + sa.Column("status", sa.String(length=32), nullable=True), + sa.Column("tier_names", sa.JSON(), nullable=True), + sa.Column("amount_cents", sa.Integer(), nullable=True), + sa.Column("currency", sa.String(length=8), nullable=True), + sa.Column( + "first_seen_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.Column( + "last_seen_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.Column("details", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_platform_membership")), + # The upsert's conflict target. Named explicitly because + # touch_membership references it by name in ON CONFLICT — an + # autogenerated name would make that call break on a rename nobody + # connected to it. + sa.UniqueConstraint( + "platform", "external_campaign_id", + name="uq_platform_membership_platform_campaign", + ), + ) + # No secondary indexes. This table holds one row per subscription — tens, + # not millions — so every query against it is a short scan and an index + # would be write cost buying nothing (#3301 removed seven of that shape). + # The unique constraint above already backs the only lookup that matters. + + +def downgrade() -> None: + op.drop_table("platform_membership") diff --git a/alembic/versions/0092_synthetic_posts.py b/alembic/versions/0092_synthetic_posts.py new file mode 100644 index 0000000..dd551f5 --- /dev/null +++ b/alembic/versions/0092_synthetic_posts.py @@ -0,0 +1,92 @@ +"""Synthetic posts — FC authors a post for content that arrived as chat. + +Milestone 388, step E2. Discord is a delivery channel, not a publisher: one +message is not one post, and today every message becomes its own `post` row +competing with authored work for the same surface. This adds the three columns +that let FC group a creator's variant drop into a post it wrote itself, while +keeping that fact visible and the grouping reversible. + +## Why a flag and a back-pointer rather than a separate table + +A synthetic post has to BE a post — same row, same columns — or every existing +surface (feed, provenance, translation, attachments, series) would need a +second code path for it. `synthesized_by` marks the ones FC authored; +`absorbed_by_post_id` points a member message-post at the post that replaced +it in the feed. The members are not deleted: they remain the images' true +origin, and destroying them would make the grouping un-auditable at exactly +the moment somebody wants to check it. + +Reversal is one DELETE. `absorbed_by_post_id` is ON DELETE SET NULL, so +removing a synthetic post releases its members and they return to the feed +unaided. + +Revision ID: 0092 +Revises: 0091 +Create Date: 2026-09-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0092" +down_revision: Union[str, None] = "0091" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # No CHECK on synthesized_by (rule 36 considered and declined): there is one + # grouper today and a second would be a new VALUE, not a new invariant — + # matching source.error_type and service_seen.kind. + op.add_column("post", sa.Column("synthesized_by", sa.String(length=32), nullable=True)) + op.add_column("post", sa.Column("synthesis_details", sa.JSON(), nullable=True)) + op.add_column( + "post", sa.Column("absorbed_by_post_id", sa.Integer(), nullable=True), + ) + op.create_index( + op.f("ix_post_absorbed_by_post_id"), "post", ["absorbed_by_post_id"], + ) + # SET NULL, not CASCADE: deleting the synthetic post must RELEASE its + # members, never take them with it. The members are the real capture. + op.create_foreign_key( + "fk_post_absorbed_by_post_id_post", "post", "post", + ["absorbed_by_post_id"], ["id"], ondelete="SET NULL", + ) + + # Grouping tunables. Every one of these is operator-facing (project rule + # 25) because the quality bar here is a judgement call no test can settle: + # too greedy merges distinct pieces, too shy leaves a drop scattered. + op.add_column( + "ml_settings", + sa.Column( + "discord_grouping_enabled", sa.Boolean(), + server_default="true", nullable=False, + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "discord_group_max_distance", sa.Float(), + server_default=sa.text("0.10"), nullable=False, + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "discord_group_window_minutes", sa.Float(), + server_default=sa.text("60"), nullable=False, + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "discord_group_window_minutes") + op.drop_column("ml_settings", "discord_group_max_distance") + op.drop_column("ml_settings", "discord_grouping_enabled") + op.drop_constraint("fk_post_absorbed_by_post_id_post", "post", type_="foreignkey") + op.drop_index(op.f("ix_post_absorbed_by_post_id"), table_name="post") + op.drop_column("post", "absorbed_by_post_id") + op.drop_column("post", "synthesis_details") + op.drop_column("post", "synthesized_by") diff --git a/alembic/versions/0093_open_groupings.py b/alembic/versions/0093_open_groupings.py new file mode 100644 index 0000000..4317375 --- /dev/null +++ b/alembic/versions/0093_open_groupings.py @@ -0,0 +1,86 @@ +"""An open grouping — a synthetic post that a later drop can still join. + +Milestone 388, step E3. E2's synthetic post was sealed at creation: a creator +who added two more variants the next day started a second post. These two +columns let the group stay open and absorb the follow-up, without the post +either freezing or thrashing the feed. + +## Why openness is derived rather than stored + +There is no `closed_at` here on purpose. A group is open if it grew (or +started) within `ml_settings.discord_group_close_after_hours`, so openness is a +comparison rather than a state — which means lowering the setting closes old +groups and raising it reopens them, with nothing to repair either way. A stored +flag would need its own sweep to set it and its own repair path to ever change +the policy, for no gain. + +## Why `resurfaced_at` is separate from `last_grew_at` + +They answer different questions. `last_grew_at` is when the group last +absorbed something — it decides how long the group stays joinable and is what +the card shows. `resurfaced_at` is the FEED POSITION, advanced only when the +anti-thrash rule fires, so a group that gains one image a day updates in place +while a genuine second wave moves once. Folding them together would make every +addition a bump, which is the annoyance this step exists to avoid. + +Both are NULL on every ordinary post, so the feed's sort key can COALESCE +through `resurfaced_at` without moving anything that is not a grouping. + +Revision ID: 0093 +Revises: 0092 +Create Date: 2026-09-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0093" +down_revision: Union[str, None] = "0092" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "post", sa.Column("last_grew_at", sa.DateTime(timezone=True), nullable=True), + ) + op.add_column( + "post", sa.Column("resurfaced_at", sa.DateTime(timezone=True), nullable=True), + ) + # No index on either. The feed already sorts on an unindexed + # COALESCE(post_date, downloaded_at) expression, so adding resurfaced_at to + # that COALESCE changes nothing about how the query plans — and inventing a + # functional index here would be guessing at the fix for a cost nobody has + # measured. Measuring it is step B2's job. + + op.add_column( + "ml_settings", + sa.Column( + "discord_group_close_after_hours", sa.Float(), + server_default=sa.text("168"), nullable=False, + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "discord_group_resurface_min_images", sa.Integer(), + server_default="2", nullable=False, + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "discord_group_resurface_cooldown_hours", sa.Float(), + server_default=sa.text("24"), nullable=False, + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "discord_group_resurface_cooldown_hours") + op.drop_column("ml_settings", "discord_group_resurface_min_images") + op.drop_column("ml_settings", "discord_group_close_after_hours") + op.drop_column("post", "resurfaced_at") + op.drop_column("post", "last_grew_at") diff --git a/alembic/versions/0094_post_association.py b/alembic/versions/0094_post_association.py new file mode 100644 index 0000000..09f093f --- /dev/null +++ b/alembic/versions/0094_post_association.py @@ -0,0 +1,124 @@ +"""post_association — "this Patreon post announced that Discord drop". + +Milestone 388, step E5. + +Two of the operator's artists post a deliberately cropped fragment on Patreon +to signal that the real thing has landed in their Discord. This table holds the +proposed and accepted links between the announcement and the drop. + +Directional and confirm-only. The pair is asymmetric (the teaser announces the +drop, not the reverse), the two posts are never merged (the creator published +twice, deliberately — flattening that hides the behaviour being modelled), and +nothing is linked until the operator accepts, following the FC-6.3 series +matcher. A wrongly-asserted association tells them two different pieces are +one, which is worse than no link at all. + +Dismissed rows are KEPT. The row is what remembers the rejection, and +re-proposing a rejected pair on every scan is what makes a review queue get +ignored. + +Revision ID: 0094 +Revises: 0093 +Create Date: 2026-09-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0094" +down_revision: Union[str, None] = "0093" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "post_association", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("announcement_post_id", sa.Integer(), nullable=False), + sa.Column("payload_post_id", sa.Integer(), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.Column("signals", sa.JSON(), nullable=True), + # No CHECK on status (rule 36 considered and declined), matching + # series_suggestion.status — the same review-queue vocabulary, and the + # same check-existing-enums lesson. + sa.Column( + "status", sa.String(length=16), server_default="pending", nullable=False, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_post_association")), + # CASCADE on both sides: an association to a post that no longer exists + # is not a fact worth keeping, and E3's reversal path (delete the + # grouping) must not leave a dangling proposal behind. + sa.ForeignKeyConstraint( + ["announcement_post_id"], ["post.id"], ondelete="CASCADE", + name=op.f("fk_post_association_announcement_post_id_post"), + ), + sa.ForeignKeyConstraint( + ["payload_post_id"], ["post.id"], ondelete="CASCADE", + name=op.f("fk_post_association_payload_post_id_post"), + ), + sa.UniqueConstraint( + "announcement_post_id", "payload_post_id", + name="uq_post_association_pair", + ), + ) + op.create_index( + op.f("ix_post_association_announcement_post_id"), + "post_association", ["announcement_post_id"], + ) + op.create_index( + op.f("ix_post_association_payload_post_id"), + "post_association", ["payload_post_id"], + ) + op.create_index( + op.f("ix_post_association_status"), "post_association", ["status"], + ) + + op.add_column( + "import_settings", + sa.Column( + "discord_link_enabled", sa.Boolean(), + server_default="true", nullable=False, + ), + ) + # 0.60 sits ABOVE the largest single signal weight on purpose — see + # post_association_service.WEIGHTS. That is what makes "time proximity + # alone is never sufficient" arithmetic rather than aspirational. + op.add_column( + "import_settings", + sa.Column( + "discord_link_threshold", sa.Float(), + server_default="0.60", nullable=False, + ), + ) + op.add_column( + "import_settings", + sa.Column( + "discord_link_window_hours", sa.Float(), + server_default="24", nullable=False, + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "discord_link_window_hours") + op.drop_column("import_settings", "discord_link_threshold") + op.drop_column("import_settings", "discord_link_enabled") + op.drop_index(op.f("ix_post_association_status"), table_name="post_association") + op.drop_index( + op.f("ix_post_association_payload_post_id"), table_name="post_association", + ) + op.drop_index( + op.f("ix_post_association_announcement_post_id"), table_name="post_association", + ) + op.drop_table("post_association") diff --git a/alembic/versions/0095_membership_sync.py b/alembic/versions/0095_membership_sync.py new file mode 100644 index 0000000..f180b01 --- /dev/null +++ b/alembic/versions/0095_membership_sync.py @@ -0,0 +1,63 @@ +"""membership_sync — whether the roster actually synced, and when. + +Milestone 387, step C3. + +`platform_membership` (0091) records what was SEEN. This records whether +looking happened at all — a different fact, and the one that makes an empty +roster readable. + +Without it, three situations collapse into one: the account subscribes to +nothing, the sweep never ran, or the sweep failed. All three leave zero rows +in `platform_membership`. "You are tracking 12 sources you no longer subscribe +to" is correct in the first case and an invitation to cancel things the +operator is actively paying for in the other two, which is why C4 gates its +CONCLUSIONS on `last_success_at` rather than merely displaying it. + +Two timestamps rather than one, deliberately: `last_attempt_at` moves every +run, `last_success_at` only on a clean walk, and the gap between them is what +lets the UI say "last synced 3 days ago, tried 20 minutes ago, failing". + +Revision ID: 0095 +Revises: 0094 +Create Date: 2026-09-11 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0095" +down_revision: Union[str, None] = "0094" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "membership_sync", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_count", sa.Integer(), nullable=True), + # No CHECK: this carries an exception class name, and the vocabulary is + # whatever the client raises — same call as source.error_type. + sa.Column("last_error_type", sa.String(length=64), nullable=True), + sa.Column("last_error_message", sa.Text(), nullable=True), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_membership_sync")), + # The upsert's conflict target, named explicitly because the service + # references it by name in ON CONFLICT. + sa.UniqueConstraint("platform", name="uq_membership_sync_platform"), + ) + # No secondary indexes: one row per platform, so every read is a short scan + # and an index would be write cost buying nothing (#3301 removed seven of + # that shape). Same reasoning as platform_membership in 0091. + + +def downgrade() -> None: + op.drop_table("membership_sync") diff --git a/alembic/versions/0096_artist_membership_suggestion.py b/alembic/versions/0096_artist_membership_suggestion.py new file mode 100644 index 0000000..431bac2 --- /dev/null +++ b/alembic/versions/0096_artist_membership_suggestion.py @@ -0,0 +1,106 @@ +"""artist_membership_suggestion — proposing that a creator and a membership match. + +Milestone 388, step E4. + +## What this migration deliberately does NOT add + +No association table between Artist and Source, and no schema change to either. +E4's first job was to verify what was actually missing, and the answer was +neither the model nor the flows: `Source.artist_id` is a plain FK so many +sources per artist already works, `POST /api/sources` already takes an +`artist_id`, the add-source dialog already attaches to an EXISTING artist, and +`SourceService.reassign` already moves a source between artists with post and +image re-attribution. Building a parallel association table for a relationship +the schema already expresses would have been the mistake rule 28 names. + +What was missing is the SUGGESTION, and that is all this table holds. + +Accepting a suggestion adds a SOURCE under the existing artist — it never +merges two artists. Adding a source is trivially undone; a wrong merge silently +mixes two creators' work and corrupts tagging, series and provenance with +nothing left to separate them by. + +Revision ID: 0096 +Revises: 0095 +Create Date: 2026-09-11 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0096" +down_revision: Union[str, None] = "0095" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "artist_membership_suggestion", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform_membership_id", sa.Integer(), nullable=False), + sa.Column("artist_id", sa.Integer(), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.Column("signals", sa.JSON(), nullable=True), + # No CHECK on status (rule 36 considered and declined), matching + # series_suggestion and post_association — the same review-queue + # vocabulary and the same check-existing-enums lesson. + sa.Column( + "status", sa.String(length=16), server_default="pending", nullable=False, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_artist_membership_suggestion")), + # CASCADE both ways: a suggestion about a membership or an artist that + # no longer exists is not a fact worth keeping, and a dangling proposal + # would render as a broken row in the review queue. + sa.ForeignKeyConstraint( + ["platform_membership_id"], ["platform_membership.id"], + ondelete="CASCADE", + name=op.f("fk_artist_membership_suggestion_membership"), + ), + sa.ForeignKeyConstraint( + ["artist_id"], ["artist.id"], ondelete="CASCADE", + name=op.f("fk_artist_membership_suggestion_artist_id_artist"), + ), + sa.UniqueConstraint( + "platform_membership_id", "artist_id", + name="uq_artist_membership_suggestion_pair", + ), + ) + op.create_index( + op.f("ix_artist_membership_suggestion_platform_membership_id"), + "artist_membership_suggestion", ["platform_membership_id"], + ) + op.create_index( + op.f("ix_artist_membership_suggestion_artist_id"), + "artist_membership_suggestion", ["artist_id"], + ) + op.create_index( + op.f("ix_artist_membership_suggestion_status"), + "artist_membership_suggestion", ["status"], + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_artist_membership_suggestion_status"), + table_name="artist_membership_suggestion", + ) + op.drop_index( + op.f("ix_artist_membership_suggestion_artist_id"), + table_name="artist_membership_suggestion", + ) + op.drop_index( + op.f("ix_artist_membership_suggestion_platform_membership_id"), + table_name="artist_membership_suggestion", + ) + op.drop_table("artist_membership_suggestion") diff --git a/alembic/versions/0097_disable_retired_platform_sources.py b/alembic/versions/0097_disable_retired_platform_sources.py new file mode 100644 index 0000000..54f3ed5 --- /dev/null +++ b/alembic/versions/0097_disable_retired_platform_sources.py @@ -0,0 +1,68 @@ +"""Disable sources on retired platforms, so the scheduler stops selecting them. + +Milestone #406, phase 1 (switch pixiv off). Rule #171 records the scope decision. + +## Why this is a migration and not a button + +The live instance had one pixiv source still ENABLED when pixiv was retired +(read 2026-09-13, step 1) even though the operator believed it gone. Unregistering +a platform removes it from code; it does not touch the `source` rows that name it. +Left enabled, that row keeps being picked by the scheduler every interval, and +`download_backends` now refuses it with `unsupported_url` — forever, as a +climbing failure count on a source the operator has already given up. + +A migration reaches the live instance on deploy without depending on anyone +finding the row and clicking it. The `run_download` guard is what makes a stale +enabled row SAFE; this is what makes it QUIET. + +## Deliberately NOT done here + +- **No rows are deleted.** Deleting a source sets its posts' `source_id` to NULL + (FK `ON DELETE SET NULL`), and `uq_post_artist_external_id_null_source` can + reject that if a source-less copy of one of those posts already exists. That + needs checking against real data first, which is phase 2's job (step 6). A + disable cannot collide with anything. +- **No posts or images are touched.** The art stays. +- **deviantart is included** because #3069 retired it and nothing disabled its + rows either. The read found none, so for it this is a no-op — written anyway, + so the statement names every retired platform rather than just the latest one. + +## Hardcoded platform names + +A migration is a record of one event, frozen in time, so it names the platforms +it acted on rather than importing today's registry — the registry will keep +changing and this revision must not. + +Revision ID: 0097 +Revises: 0096 +Create Date: 2026-09-13 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0097" +down_revision: Union[str, None] = "0096" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Clears the failure state the same way `SourceService.update` does when a + # source is disabled through the app (issue #1285), so a retired source + # does not keep showing as failing after it stops being polled. A disable + # done here and one done by clicking must leave identical rows. + op.execute( + "UPDATE source SET enabled = false, last_error = NULL, " + "error_type = NULL, consecutive_failures = 0 " + "WHERE enabled AND platform IN ('pixiv', 'deviantart')" + ) + + +def downgrade() -> None: + # Irreversible by design: which of these rows were enabled before is not + # recorded, and re-enabling every retired-platform source would resume + # polling services the product no longer supports. Rule #22 owes no + # migration story back to a dropped platform. + pass diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index 5660090..ff3483f 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -48,6 +48,17 @@ _EDITABLE = ( "process_conflict_threshold", "embedder_model_name", "embedder_model_version", + # Discord drop grouping (#388 E2). Operator-facing because the quality bar + # is a judgement no test can settle: too greedy merges distinct pieces, too + # shy leaves a drop scattered. + "discord_grouping_enabled", + "discord_group_max_distance", + "discord_group_window_minutes", + # E3: how long a grouping stays open, and the anti-thrash rule that keeps + # a growing one from monopolising the feed. + "discord_group_close_after_hours", + "discord_group_resurface_min_images", + "discord_group_resurface_cooldown_hours", *_DETECTOR_FIELDS, ) @@ -148,6 +159,24 @@ def _validate(p: dict) -> str | None: return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}" if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0): return "process_conflict_threshold must be between 0 and 1" + # Discord drop grouping (#388 E2). max_distance is a cosine DISTANCE, so + # unlike the *_threshold family above it is not on the auto-apply scale: + # 0 is identical and 1 is unrelated, and both ends are legal. The upper + # bound is 1.0 rather than AUTO_APPLY_THRESHOLD_MAX for that reason. + if not (0.0 <= float(p["discord_group_max_distance"]) <= 1.0): + return "discord_group_max_distance must be between 0 and 1" + if float(p["discord_group_window_minutes"]) <= 0: + return "discord_group_window_minutes must be > 0" + # A group must stay open at least as long as the drop window it was cut + # with, or the joiner could never reach a message the grouper deferred — + # the two would fight, and the symptom (drops that never grow) would look + # like the predicate failing rather than a settings contradiction. + if float(p["discord_group_close_after_hours"]) * 60 < float(p["discord_group_window_minutes"]): + return "discord_group_close_after_hours must be at least the drop window" + if int(p["discord_group_resurface_min_images"]) < 1: + return "discord_group_resurface_min_images must be >= 1" + if float(p["discord_group_resurface_cooldown_hours"]) < 0: + return "discord_group_resurface_cooldown_hours must be >= 0" # Embedder model swap (#1190): both must be non-empty. Changing them means a # different embedding space — the operator must re-embed + retrain after. for key in ("embedder_model_name", "embedder_model_version"): diff --git a/backend/app/api/posts.py b/backend/app/api/posts.py index b6334db..2823368 100644 --- a/backend/app/api/posts.py +++ b/backend/app/api/posts.py @@ -5,6 +5,8 @@ from quart import Blueprint, jsonify, request from ..extensions import get_session from ..models import ImportSettings, Post from ..services import interpreter_client as ic +from ..services.post_association_service import PostAssociationService +from ..services.post_association_service import rescan as association_rescan from ..services.post_feed_service import PostFeedService from ..services.source_service import KNOWN_PLATFORMS from ..utils.text import html_to_plain @@ -165,3 +167,46 @@ async def set_translation_override(post_id: int): "translated_source_lang": post.translated_source_lang, "applied": applied, }) + + +# --- #388 E5: the announcement review queue ------------------------------- +# +# Confirm-only, following the series-suggestion routes (api/tags.py). Nothing +# here links anything on its own: the matcher proposes, the operator decides. + + +@posts_bp.route("/associations", methods=["GET"]) +async def list_associations(): + async with get_session() as session: + return jsonify({"items": await PostAssociationService(session).list_pending()}) + + +@posts_bp.route("/associations//accept", methods=["POST"]) +async def accept_association(association_id: int): + async with get_session() as session: + result = await PostAssociationService(session).accept(association_id) + if result is None: + return _bad("association not found", 404) + await session.commit() + return jsonify(result) + + +@posts_bp.route("/associations//dismiss", methods=["POST"]) +async def dismiss_association(association_id: int): + async with get_session() as session: + result = await PostAssociationService(session).dismiss(association_id) + if result is None: + return _bad("association not found", 404) + await session.commit() + return jsonify(result) + + +@posts_bp.route("/associations/rescan", methods=["POST"]) +async def rescan_associations(): + """Manual re-scan. The beat sweep only looks at recent posts (a pair has to + be within the window to exist at all); this is the button for a first run + over a library that predates the feature.""" + async with get_session() as session: + result = await association_rescan(session) + await session.commit() + return jsonify(result) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index ea76989..751e60e 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -39,6 +39,10 @@ _EDITABLE_FIELDS = ( "download_failure_warning_threshold", "series_suggest_enabled", "series_suggest_threshold", + # #388 E5 — the announcement matcher (Patreon teaser ↔ Discord drop). + "discord_link_enabled", + "discord_link_threshold", + "discord_link_window_hours", "extdl_mega_enabled", "extdl_gdrive_enabled", "extdl_mediafire_enabled", @@ -150,6 +154,22 @@ async def update_import_settings(): return jsonify( {"error": "series_suggest_threshold must be a number in [0, 1]"} ), 400 + if "discord_link_enabled" in body and not isinstance( + body["discord_link_enabled"], bool + ): + return jsonify({"error": "discord_link_enabled must be a boolean"}), 400 + if "discord_link_threshold" in body: + v = body["discord_link_threshold"] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: + return jsonify( + {"error": "discord_link_threshold must be a number in [0, 1]"} + ), 400 + if "discord_link_window_hours" in body: + v = body["discord_link_window_hours"] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v <= 0: + return jsonify( + {"error": "discord_link_window_hours must be a positive number"} + ), 400 if "wip_title_tagging_enabled" in body and not isinstance( body["wip_title_tagging_enabled"], bool ): diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 2463ea2..4438c4c 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -1,10 +1,15 @@ """FC-3a: CRUD over Source rows. FC-3c adds POST //check.""" from quart import Blueprint, jsonify, request -from sqlalchemy import select +from sqlalchemy import func, select from ..extensions import get_session -from ..models import DownloadEvent, Source +from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source +from ..services.artist_membership_service import ArtistMembershipService +from ..services.artist_membership_service import rescan as membership_rescan +from ..services.artist_service import ArtistService +from ..services.membership_reconcile import reconcile_all +from ..services.membership_roster import roster_is_fresh, source_for_membership from ..services.scheduler_service import active_platform_cooldowns, scheduler_status from ..services.source_service import ( KNOWN_PLATFORMS, @@ -288,3 +293,163 @@ async def check_source(source_id: int): download_source.delay(source_id) return jsonify({"download_event_id": event_id, "status": "pending"}), 202 + + +# --- #387 C3: the membership roster's sync state -------------------------- +# +# Rule 164's visibility requirement lives here. A roster that failed to sync, +# or never has, must be DISTINGUISHABLE from an account that subscribes to +# nothing — otherwise the reconciliation this unlocks would tell the operator +# to cancel sources they are actively paying for. + + +@sources_bp.route("/membership-sync", methods=["GET"]) +async def membership_sync_status(): + async with get_session() as session: + rows = (await session.execute(select(MembershipSync))).scalars().all() + counts = dict( + (await session.execute( + select(PlatformMembership.platform, func.count()) + .group_by(PlatformMembership.platform) + )).all() + ) + return jsonify({"platforms": [ + { + "platform": r.platform, + "last_attempt_at": r.last_attempt_at.isoformat() if r.last_attempt_at else None, + # NULL here means NEVER, and the UI must say so in words. Rendering + # it as 0 or as "-" is the exact conflation this endpoint exists to + # prevent. + "last_success_at": r.last_success_at.isoformat() if r.last_success_at else None, + "last_count": r.last_count, + "last_error_type": r.last_error_type, + "last_error_message": r.last_error_message, + # Whether a CONCLUSION may be drawn from this roster — not merely + # whether it looks recent. C4 gates on this, and it is computed + # server-side so no caller can forget to. + "fresh": roster_is_fresh(r), + "known_memberships": counts.get(r.platform, 0), + } + for r in sorted(rows, key=lambda r: r.platform) + ]}) + + +@sources_bp.route("/membership-sync", methods=["POST"]) +async def trigger_membership_sync(): + """Run the roster sweep now. + + The beat schedule runs daily, which is right for a billing-cycle fact but + far too slow when the operator has just connected a credential and wants to + see whether it works. Queued rather than run inline: it crosses the network + to an external service and the request path is not where that belongs. + """ + from ..tasks.maintenance import sync_memberships + + sync_memberships.delay() + return jsonify({"queued": True}) + + +# --- #388 E4: creator/membership suggestions ------------------------------ +# +# Confirm-only. Accepting ADDS A SOURCE under the existing artist — it never +# merges two artists, because adding a source is trivially undone and a wrong +# merge silently mixes two creators' work with nothing left to separate them by. + + +@sources_bp.route("/membership-suggestions", methods=["GET"]) +async def list_membership_suggestions(): + async with get_session() as session: + return jsonify({"items": await ArtistMembershipService(session).list_pending()}) + + +@sources_bp.route("/membership-suggestions//accept", methods=["POST"]) +async def accept_membership_suggestion(sid: int): + async with get_session() as session: + result = await ArtistMembershipService(session).accept(sid) + if result is None: + return _bad("suggestion_not_found", status=404) + await session.commit() + return jsonify(result) + + +@sources_bp.route("/membership-suggestions//dismiss", methods=["POST"]) +async def dismiss_membership_suggestion(sid: int): + async with get_session() as session: + result = await ArtistMembershipService(session).dismiss(sid) + if result is None: + return _bad("suggestion_not_found", status=404) + await session.commit() + return jsonify(result) + + +@sources_bp.route("/membership-suggestions/rescan", methods=["POST"]) +async def rescan_membership_suggestions(): + async with get_session() as session: + result = await membership_rescan(session) + await session.commit() + return jsonify(result) + + +# --- #387 C4: reconciling the roster against what FC actually tracks ------- +# +# Asymmetric on purpose. The "you subscribe but FC doesn't follow it" direction +# carries a per-row action, because adding a source is the reversible half. The +# "FC follows it but your roster doesn't show it" direction is REPORT ONLY by +# the operator's decision (2026-09-11): it says what it sees and links to the +# Subscriptions row, and offers no one-click disable. + + +@sources_bp.route("/reconciliation", methods=["GET"]) +async def reconciliation(): + async with get_session() as session: + return jsonify(await reconcile_all(session)) + + +@sources_bp.route("/reconciliation/adopt", methods=["POST"]) +async def adopt_membership(): + """Start tracking a creator the roster says the account already pays for. + + One row, one click, never a sweep side effect: adding a source commits disk, + worker time and rate budget, and unwinding it means deleting files. + """ + body = await request.get_json() + if not isinstance(body, dict): + return _bad("invalid_body", status=400) + membership_id = body.get("membership_id") + if not isinstance(membership_id, int): + return _bad("membership_id_required", status=400) + + async with get_session() as session: + membership = await session.get(PlatformMembership, membership_id) + if membership is None: + return _bad("membership_not_found", status=404) + if not membership.url: + return _bad("membership_has_no_url", status=400) + + existing = await source_for_membership(session, membership) + if existing is not None: + # The operator got there by another route between the page load and + # the click. That is them being ahead of us, not an error. + return jsonify({"already_tracked": existing.id}) + + # The sweep already captured the creator's real display name, so the + # artist gets its true name with NO lookup on the request path. Task + # #1293 asked for `resolve_display_name` here; the roster satisfies that + # concern earlier in the pipeline than #1293 expected, which also keeps + # this route off the network entirely (rule 164). The vanity is the + # fallback, never the preferred value. + name = membership.display_name or membership.vanity_or_none() + if not name: + return _bad("membership_has_no_name", status=400) + + artist, _created = await ArtistService(session).find_or_create(name) + try: + record = await SourceService(session).create( + artist_id=artist.id, + platform=membership.platform, + url=membership.url, + ) + except DuplicateSourceError as exc: + return jsonify({"already_tracked": exc.existing_id}) + artist_id = artist.id + return jsonify({"source_id": record.id, "artist_id": artist_id}), 201 diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index d8983ed..b702fc6 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -200,6 +200,26 @@ def make_celery() -> Celery: "task": "backend.app.tasks.maintenance.snapshot_head_metrics", "schedule": 86400.0, }, + "group-discord-drops-hourly": { + "task": "backend.app.tasks.maintenance.group_discord_drops", + "schedule": 3600.0, # hourly. Not daily: the grouping signal is + # the SigLIP embedding, which lands asynchronously AFTER import + # (#388 E2), so this sweep is what picks up a drop once its + # vectors have caught up. No-op unless discord_grouping_enabled. + }, + "match-post-associations-hourly": { + "task": "backend.app.tasks.maintenance.match_post_associations", + "schedule": 3600.0, # hourly, and AFTER the grouper's own cadence + # by construction: a pair cannot be proposed until the drop it + # points at exists as a grouping (#388 E5). No-op unless + # discord_link_enabled. + }, + "sync-memberships-daily": { + "task": "backend.app.tasks.maintenance.sync_memberships", + "schedule": 86400.0, # daily — memberships change on a BILLING + # cycle, not a download cadence (#387 C3). No-op per platform + # when the client lacks the seam or no credential exists. + }, "integrity-verify-weekly": { "task": "backend.app.tasks.maintenance.verify_integrity", "schedule": 604800.0, # weekly diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index fe4bf54..9e085f4 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -2,6 +2,7 @@ from .app_setting import AppSetting from .artist import Artist +from .artist_membership_suggestion import ArtistMembershipSuggestion from .artist_visit import ArtistVisit from .backup_run import BackupRun from .base import Base @@ -21,12 +22,15 @@ from .import_batch import ImportBatch from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun +from .membership_sync import MembershipSync from .ml_settings import MLSettings from .patreon_failed_media import PatreonFailedMedia from .patreon_seen_media import PatreonSeenMedia from .pixiv_failed_media import PixivFailedMedia from .pixiv_seen_media import PixivSeenMedia +from .platform_membership import PlatformMembership from .post import Post +from .post_association import PostAssociation from .post_attachment import PostAttachment, attachment_download_url from .presentation_review import PresentationReview from .series_chapter import SeriesChapter @@ -47,6 +51,7 @@ __all__ = [ "Base", "AppSetting", "Artist", + "ArtistMembershipSuggestion", "ArtistVisit", "BackupRun", "Source", @@ -58,12 +63,14 @@ __all__ = [ "SubscribeStarFailedMedia", "SubscribeStarSeenMedia", "Post", + "PostAssociation", "PostAttachment", "attachment_download_url", "PresentationReview", "SeriesChapter", "SeriesPage", "SeriesSuggestion", + "PlatformMembership", "ServiceSeen", "ImageRecord", "ImageProvenance", @@ -78,6 +85,7 @@ __all__ = [ "ImportTask", "ImportSettings", "LibraryAuditRun", + "MembershipSync", "MLSettings", "HeadAutoApplyRun", "HeadMetric", diff --git a/backend/app/models/artist_membership_suggestion.py b/backend/app/models/artist_membership_suggestion.py new file mode 100644 index 0000000..ebc8f54 --- /dev/null +++ b/backend/app/models/artist_membership_suggestion.py @@ -0,0 +1,83 @@ +"""artist_membership_suggestion — "this creator and that membership are the same". + +Milestone 388, step E4. + +## What was NOT needed here + +E4's first job was to check what is actually missing, and the answer was: not +the schema, and not the flows. `Source.artist_id` is a plain FK, so many +sources per artist is already the data model; `POST /api/sources` already takes +an `artist_id`; the add-source dialog already has an artist autocomplete that +attaches to an EXISTING artist; and `SourceService.reassign` already moves a +source between artists WITH post and image re-attribution. A sweep for +one-source-per-artist assumptions found only `func.count()` calls, which are +the opposite of assuming one. + +So no parallel association table was built for a relationship the schema +already expresses (rule 28). What was missing is the SUGGESTION — FC proposing +the link from the roster instead of waiting to be told. + +## Confirm-only, and what "accept" actually does + +Accepting adds a SOURCE for the membership's platform under the artist that +already has the other channel. It does NOT merge two artists. That distinction +is the whole safety margin: adding a source is trivially undone, whereas a +wrong artist merge silently mixes two creators' work and corrupts tagging, +series and provenance downstream — with nothing left to tell them apart by. + +Dismissed rows are kept, not deleted, for the same reason as every other review +queue here: the row is what remembers the rejection, and re-proposing a +rejected pair on every scan is what makes a queue get ignored. +""" + +from datetime import datetime + +from sqlalchemy import ( + JSON, + DateTime, + Float, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class ArtistMembershipSuggestion(Base): + __tablename__ = "artist_membership_suggestion" + __table_args__ = ( + UniqueConstraint( + "platform_membership_id", "artist_id", + name="uq_artist_membership_suggestion_pair", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + platform_membership_id: Mapped[int] = mapped_column( + ForeignKey("platform_membership.id", ondelete="CASCADE"), + nullable=False, index=True, + ) + artist_id: Mapped[int] = mapped_column( + ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True + ) + + score: Mapped[float] = mapped_column(Float, nullable=False) + # Per-signal strengths as scored. Without it, "why was this suggested" is + # unanswerable the moment a weight or the threshold moves. + signals: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # pending | linked | dismissed. Plain String, no CHECK — same call as + # series_suggestion.status and post_association.status. + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending", index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + server_default=func.now(), onupdate=func.now(), + ) diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index 83ae9c2..0ed2e8b 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -97,6 +97,33 @@ class ImportSettings(Base): server_default="0.5", ) + # Milestone 388 E5 — the announcement matcher: "this Patreon post announced + # that Discord drop". Lives here rather than in MLSettings, with the series + # matcher it is modelled on, because it runs no inference: the signals are + # time proximity and whether the post says so. + discord_link_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, + server_default="true", + ) + # The weighted-score cut-off. 0.60 is not arbitrary: it is deliberately set + # ABOVE the largest single signal weight, which is what makes "time + # proximity alone must never be sufficient" an ARITHMETIC property rather + # than a hope. On a busy day an artist posts several times; if proximity + # could carry a pair by itself, every one of those days would produce false + # pairs and the review queue would be abandoned. See + # post_association_service.WEIGHTS — a guard test pins the relationship. + discord_link_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.60, + server_default="0.60", + ) + # How far apart the announcement and the drop may be. The Patreon post + # exists IN ORDER TO announce the drop, so they are minutes-to-hours apart; + # a day is generous and still excludes "same week". + discord_link_window_hours: Mapped[float] = mapped_column( + Float, nullable=False, default=24.0, + server_default="24", + ) + # #830 off-platform file-host downloads — per-host enable lever (default on, # rule #26). Column names are extdl__enabled so the worker reads them # via getattr(settings, f"extdl_{host}_enabled", True). diff --git a/backend/app/models/membership_sync.py b/backend/app/models/membership_sync.py new file mode 100644 index 0000000..d624418 --- /dev/null +++ b/backend/app/models/membership_sync.py @@ -0,0 +1,77 @@ +"""membership_sync — did the roster actually sync, and when. + +Milestone 387, step C3. + +`platform_membership` records what was SEEN. This records whether looking +happened at all, and that is a different fact — the one that makes an empty +roster readable. + +## Why this table has to exist + +Without it, three very different situations are one indistinguishable state: + +* the account genuinely subscribes to nothing, +* the sweep has never run, +* the sweep ran and failed. + +All three produce zero rows in `platform_membership`. Telling the operator +"you are tracking 12 sources you do not subscribe to" is correct in the first +case and catastrophic in the other two — it is an invitation to cancel things +they are actively paying for. C4 must therefore gate its CONCLUSIONS on +`last_success_at`, not merely display it. + +`MAX(platform_membership.last_seen_at)` was the tempting shortcut and does not +work: it cannot distinguish "synced fine, found nothing" from "never synced". +`task_run` was the other candidate and is worse — its retention prunes ok rows +after 24h, so a sweep that last succeeded three days ago would leave no trace +at all. + +## Separate attempt and success timestamps, deliberately + +`last_attempt_at` moves every run; `last_success_at` moves only on a clean +walk. The GAP between them is the staleness signal, and keeping them apart is +what lets the UI say "last synced 3 days ago, last tried 20 minutes ago, +failing" — which is a different message from either half alone. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class MembershipSync(Base): + __tablename__ = "membership_sync" + __table_args__ = ( + UniqueConstraint("platform", name="uq_membership_sync_platform"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + platform: Mapped[str] = mapped_column(String(64), nullable=False) + + # Moves on EVERY run, success or not — so "we are trying" is visible even + # while "we are succeeding" is not. + last_attempt_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + # Moves only on a COMPLETE walk. This is the freshness signal C4 gates its + # conclusions on; NULL means never — which must never be rendered as zero. + last_success_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + # How many memberships the last SUCCESSFUL walk saw. Paired with + # last_success_at so "0" is only ever readable as a real zero. + last_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + # Cleared on success. Plain String, no CHECK — this carries an exception + # class name (PatreonAuthError, PatreonDriftError, ...) and the vocabulary + # is whatever the client raises, exactly as source.error_type works. + last_error_type: Mapped[str | None] = mapped_column(String(64), nullable=True) + last_error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + server_default=func.now(), onupdate=func.now(), + ) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 4705d1a..021a56f 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -252,6 +252,69 @@ class MLSettings(Base): Integer, nullable=False, default=64, server_default="64", ) + # -- Discord drop grouping (milestone 388) ----------------------------- + # FC authors a post out of a creator's variant drop. The predicate is three + # axes ANDed together, and the time one does the real work: SIMILARITY + # ALONE OVER-GROUPS. Any two pieces of the same character by the same + # artist sit close in SigLIP space, so a cosine-only rule collapses a month + # of one character into a single "post". What makes a variant set a set is + # that it was dropped TOGETHER. + discord_grouping_enabled: Mapped[bool] = mapped_column( + # ON by default, matching the operator's standing opt-OUT preference for + # automatic behaviour (2026-06-29, recorded on the head/ccip auto-apply + # switches). Safe to default on because the act is reversible by one + # DELETE: removing a synthetic post un-absorbs its members. + Boolean, nullable=False, default=True, + server_default="true", + ) + # Cosine DISTANCE, not similarity — this is the units gallery_service's + # `cosine_distance` already speaks, and converting at the query site is a + # step to get backwards. Lower = stricter. 0.10 is deliberately TIGHT: the + # two failure modes are not symmetric. Grouping too shy leaves a drop + # scattered, which is visible and fixable by raising this; grouping too + # greedy merges distinct pieces into a post that claims they belong + # together, which is the failure that would discredit the feature. + discord_group_max_distance: Mapped[float] = mapped_column( + Float, nullable=False, default=0.10, + server_default=text("0.10"), + ) + # The gap that ENDS a drop, measured between CONSECUTIVE messages rather + # than from the first — an artist trickling variants out over an evening is + # one drop, and a window anchored on the first message would cut it in half + # at an arbitrary point. + discord_group_window_minutes: Mapped[float] = mapped_column( + Float, nullable=False, default=60.0, + server_default=text("60"), + ) + # How long a synthetic post keeps accepting new members (#388 E3). This is + # NOT the drop window above: the window cuts one sweep's messages into + # drops, this decides how long a finished drop can still be REJOINED when a + # creator adds variants days later. A week by default — long enough for the + # "and here is the alt outfit" follow-up that motivated the feature, short + # enough that a group does not still be open when the same character comes + # round again months later and gets absorbed by mistake. + # + # Openness is DERIVED from this, not stored: a group is open if it grew (or + # started) within this period. So lowering it closes old groups and raising + # it reopens them, which is comprehensible and reversible — the alternative, + # a stored closed_at, would need its own repair path to ever change. + discord_group_close_after_hours: Mapped[float] = mapped_column( + Float, nullable=False, default=168.0, + server_default=text("168"), + ) + # Anti-thrash (#388 E3). An updated post SHOULD be visible — that is the + # point of keeping it open — but a group gaining one image a day must not + # monopolise the feed. Growth smaller than this never moves the post, and + # no group moves more than once per cooldown, so a drip-feed updates in + # place while a real second wave resurfaces exactly once. + discord_group_resurface_min_images: Mapped[int] = mapped_column( + Integer, nullable=False, default=2, + server_default="2", + ) + discord_group_resurface_cooldown_hours: Mapped[float] = mapped_column( + Float, nullable=False, default=24.0, + server_default=text("24"), + ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/platform_membership.py b/backend/app/models/platform_membership.py new file mode 100644 index 0000000..4e3de1a --- /dev/null +++ b/backend/app/models/platform_membership.py @@ -0,0 +1,152 @@ +"""platform_membership — the learned roster of what the account actually pays for. + +Milestone 387, phase C. FabledCurator knows which creators it has been TOLD to +follow (`source`), and nothing about which ones the operator is actually +subscribed to. Those two sets drift in both directions and the app cannot +currently see either drift: + +* A subscription the operator pays for that FC does not track is content they + believe they are archiving and are not. +* A source FC keeps walking after the subscription lapsed is requests spent on + a wall, reported as a creator who has gone quiet. + +This table is the memory that makes both visible — every membership the account +has been observed to hold, and when it was last seen. + +## Why a learned roster rather than a live lookup + +Same reasoning as `service_seen` (milestone 365), and the same shape: an +absence is only observable against a record of presence. A membership that +stops appearing in a sweep is the signal — "you were subscribed to this, now +you aren't" — and there is nowhere to read that from a live call, because a +live call returns what IS, never what stopped being. + +It also means the reconciliation surface keeps working when Patreon is +unreachable, degraded to a stale roster with a visible age rather than an empty +page (rule 164). + +## Roster truth, NOT per-post truth + +The single most important thing about this table: `tier_names` says which tiers +the account holds. It does **not** say which posts those tiers unlock. A +creator can gate a post behind an access rule that maps onto no tier name at +all. + +`current_user_can_view` — read per post by `patreon_client.post_is_gated` — is +the authoritative signal, and phase A already turned it into a durable +per-source state. This roster EXPLAINS that state ("you are no longer a patron" +vs "your tier doesn't cover these posts"). It must never be used to decide +whether to fetch something. Getting that backwards would make FC silently stop +fetching content the operator is paying for, which is the worst failure +available in this milestone. + +## status is a plain String, and deliberately the platform's own word + +Not a Postgres ENUM, not CHECK-gated — matching `service_seen.kind`, +`gpu_job.status` and `source.error_type`. Two reasons, and the first is the +real one: + +1. **The vocabulary is not ours to invent.** Patreon says `active_patron` / + `former_patron` / `declined_patron`; SubscribeStar and FANBOX will say + something else. Storing each platform's own word verbatim and mapping to + FC's meaning at the READ site keeps this table a record of what was + observed rather than a lossy translation of it. A lowest-common-denominator + enum picked before any platform has been characterised (step C0) would be a + guess baked into the schema. +2. A constraint swap per new value (rule 36) would be cost with no invariant + behind it, exactly as `service_seen.kind` records. + +The service layer owns the whitelist and the mapping; the column owns the +evidence. + +## Retention: aged out, never deleted on disappearance + +A membership that stops appearing in a sweep is NOT removed. Its disappearance +is the fact the reconciliation surface reads, and deleting the row would +destroy the signal at the moment it became interesting. `last_seen_at` is what +makes "gone" decidable, and a retention policy ages rows out on time rather +than on absence. +""" + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class PlatformMembership(Base): + __tablename__ = "platform_membership" + __table_args__ = ( + # The natural key the sweep's upsert conflicts on. Named explicitly + # because `touch_membership` references it by name in ON CONFLICT. + UniqueConstraint( + "platform", "external_campaign_id", + name="uq_platform_membership_platform_campaign", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + platform: Mapped[str] = mapped_column(String(64), nullable=False) + # The platform's own id for the thing subscribed to — a Patreon campaign + # id, whatever SubscribeStar and FANBOX call theirs. Text rather than a + # bounded String: these are opaque upstream identifiers and guessing a + # ceiling for a value we do not mint is how a walk dies on a truncation. + external_campaign_id: Mapped[str] = mapped_column(Text, nullable=False) + + # For the reconciliation UI, and for matching against Source.url — the + # vanity/URL is what the two sides actually have in common. + display_name: Mapped[str | None] = mapped_column(Text, nullable=True) + url: Mapped[str | None] = mapped_column(Text, nullable=True) + + # The platform's own word. See the module docstring — this is evidence, + # not a normalised FC status. + status: Mapped[str | None] = mapped_column(String(32), nullable=True) + + # Nullable throughout: a free follow has no tier and no money attached, and + # a platform may not expose an amount at all. Absent must stay + # distinguishable from zero — "free" and "we don't know" are different + # answers to "what is this costing". + tier_names: Mapped[list | None] = mapped_column(JSON, nullable=True) + amount_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) + currency: Mapped[str | None] = mapped_column(String(8), nullable=True) + + # NEVER updated after insert. The one field that answers "has this ever + # been true", which is what makes a disappearance readable rather than + # indistinguishable from never having existed. `touch_membership` + # deliberately excludes it from the ON CONFLICT update set. + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + # The raw membership as the platform returned it, so a later question can + # be answered without re-fetching — and so a field we did not think to + # model is not lost. Displayed and never queried, like service_seen.details. + details: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + + def vanity_or_none(self) -> str | None: + """The platform's URL slug for this creator, if it can be known. + + NOT a column, and that is C1's design working as intended rather than + an omission: the roster was modelled before any platform had been + characterised, so `details` exists precisely to carry the fields we did + not know to model. The vanity turned out to be one of them (#3886), and + it is reachable without a migration. + + Falls back to the URL's last segment, which is what a vanity IS on + every platform seen so far — but only as a fallback, because the + platform's own word for it is the better answer when present. + """ + campaign = (self.details or {}).get("campaign") or {} + vanity = campaign.get("vanity") + if isinstance(vanity, str) and vanity: + return vanity + if self.url: + tail = self.url.rstrip("/").rsplit("/", 1)[-1] + return tail or None + return None diff --git a/backend/app/models/post.py b/backend/app/models/post.py index e3f8e5e..8fa07fb 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -102,3 +102,61 @@ class Post(Base): downloaded_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + + # -- Synthetic posts (milestone 388). ---------------------------------- + # Discord is a delivery CHANNEL, not a publisher: one message is not one + # post. So FC authors the post itself, grouping a creator's variant drop + # into a single row (services/discord_grouping.py). + # + # NULL for every post a creator actually wrote — which is all of them until + # a grouper runs. Non-NULL names the grouper that authored this row, and is + # the ONE flag the UI keys off to say so. The honesty rule is the whole + # point: a synthetic post must never present itself as authored, and a + # column that is absent-or-a-name makes "was this us?" answerable from the + # row rather than inferred from its shape. + # + # Plain String, no CHECK (rule 36 considered and declined) — same reasoning + # as source.error_type and service_seen.kind. There is exactly one grouper + # today; a second would be a value, not an invariant. + synthesized_by: Mapped[str | None] = mapped_column(String(32), nullable=True) + # What it was built from, so the operator can audit a grouping FC invented: + # member post ids, message count, and the thresholds in force when the + # decision was made. That last part matters — the thresholds are operator- + # tunable, so "why did it group these" is unanswerable a month later + # without recording the values that produced it. + synthesis_details: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Set on a MEMBER post, pointing at the synthetic post that absorbed it. + # The feed hides absorbed posts (they are the chat lines the synthetic post + # replaced); every other surface still reaches them by id, because they + # remain the image's true origin and the grouping has to be inspectable. + # + # Self-FK, ON DELETE SET NULL: deleting a synthetic post un-absorbs its + # members and they return to the feed on their own. That is the reversal + # path, and it is one DELETE — nothing to undo by hand. + absorbed_by_post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + # -- An OPEN grouping (milestone 388 E3) ------------------------------- + # A synthetic post is not sealed at creation: a creator who adds two more + # variants the next day extends the existing post rather than starting a + # new one. These two columns are what make that possible without the post + # either freezing or thrashing the feed. + # + # `last_grew_at` is when the group last absorbed something. It answers two + # questions: how long the group stays JOINABLE (a group closes after a + # quiet period — artists reuse characters for years, and a group left open + # forever will eventually absorb something it shouldn't), and what the card + # shows as "updated N ago". NULL means it has never grown since creation. + last_grew_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + # The feed position, and ONLY set when the anti-thrash rule fires — see + # discord_grouping.should_resurface. A group that gains one image a day + # must not sit permanently at the top of the feed, so growth updates the + # post without necessarily moving it; a genuine second wave moves it once. + # + # NULL on every ordinary post, which is why the feed's sort key can + # COALESCE through it without changing where anything else lands. + resurfaced_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/backend/app/models/post_association.py b/backend/app/models/post_association.py new file mode 100644 index 0000000..96eaa3f --- /dev/null +++ b/backend/app/models/post_association.py @@ -0,0 +1,100 @@ +"""PostAssociation — "this Patreon post announced that Discord drop". + +Milestone 388, step E5, and the point of the milestone rather than its tail. + +Two of the operator's artists post a deliberately CROPPED fragment on Patreon +to signal that the real thing has landed in their Discord. The Patreon post is +the announcement; the Discord grouping (milestone 388 E2) is the payload. This +row is the link between them. + +## Directional, and NOT a merge + +`announcement` → `payload` is asymmetric on purpose. The teaser announces the +drop; the drop does not announce the teaser, and a symmetric "related posts" +edge would lose the only thing that makes the pair interesting. + +Nor are the two collapsed into one post. The creator published twice, +deliberately, on two platforms with different audiences — flattening that +hides the very behaviour being modelled, and would destroy the operator's +ability to see that the Patreon post is a teaser at all. + +## Confirm-only, following FC-6.3 (task 737) + +`status` starts at `pending` and nothing is linked until the operator accepts. +A wrongly-asserted association tells them two different pieces are one, which +is worse than no link: no link leaves them where they already are, a wrong one +actively misinforms. Same reason the series matcher writes to a review queue +instead of filing posts on its own. + +`status` is a plain String, no CHECK — matching `series_suggestion.status`, +which records the same check-existing-enums lesson. + +## Why pHash could not do this, and the correction matters + +The original plan claimed this link was already sitting in `image_provenance` +via pHash dedup. It is not. `compute_phash` is `imagehash.phash` at +`hash_size=8` — a DCT hash over the WHOLE image, robust to rescaling and +recompression but NOT to cropping, because a crop changes the global +signature. Cross-platform provenance still links straight re-posts; it does +nothing for a cropped teaser and its full version, which is precisely the pair +the operator described. Hence a scored proposal rather than a lookup. +""" + +from datetime import datetime + +from sqlalchemy import ( + JSON, + DateTime, + Float, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class PostAssociation(Base): + __tablename__ = "post_association" + __table_args__ = ( + UniqueConstraint( + "announcement_post_id", "payload_post_id", + name="uq_post_association_pair", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + # The teaser — a real post the creator wrote (Patreon, today). + announcement_post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + # What it announced — a synthetic Discord grouping, today. CASCADE on both + # sides: an association to a post that no longer exists is not a fact worth + # keeping, and E3's reversal path (delete the grouping) should not leave a + # dangling proposal behind. + payload_post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + + score: Mapped[float] = mapped_column(Float, nullable=False) + # Per-signal strengths as scored, so a proposal stays explicable after the + # weights or the threshold are tuned. Without it, "why was this suggested" + # is unanswerable the moment anything moves. + signals: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # pending | linked | dismissed. A DISMISSED row is kept, not deleted — it + # is what stops the matcher proposing the same rejected pair on every + # subsequent scan, which is the behaviour that makes a review queue + # unusable. + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending", index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + server_default=func.now(), onupdate=func.now(), + ) diff --git a/backend/app/services/artist_membership_service.py b/backend/app/services/artist_membership_service.py new file mode 100644 index 0000000..020c673 --- /dev/null +++ b/backend/app/services/artist_membership_service.py @@ -0,0 +1,337 @@ +"""Proposing that a creator FC tracks and a membership it found are the same. + +Milestone 388, step E4. An instance of the confirm-only matcher shape +(snippet #3842), and a sibling of `post_association_service`. + +## What E4 turned out NOT to need + +The step's own first instruction was to verify before building, and the +verification said: not the schema, not the flows. `Source.artist_id` is a plain +FK so many sources per artist already works; `POST /api/sources` already takes +an `artist_id`; the add-source dialog already has an artist autocomplete that +attaches to an EXISTING artist; `SourceService.reassign` already moves a source +between artists WITH post and image re-attribution; and a sweep for +one-source-per-artist assumptions found only `func.count()` calls, which are +the opposite of assuming one. + +So the association a Discord source and a Patreon source share is already +expressible today. What was missing is FC OFFERING it. + +## Accept adds a SOURCE — it never merges artists + +The asymmetry that sets the whole posture: adding a source is trivially undone. +A wrong artist merge silently mixes two creators' work and corrupts tagging, +series and provenance downstream, with nothing left to tell the two apart by. +So the accepted action is "add the missing channel to this artist", and merging +is not offered at all. + +## The signals + +1. **Name.** The roster's `display_name` and `vanity`, slugified, against the + artist's `slug`. Graded rather than boolean — an exact match is strong + evidence, a containment match is a hint. +2. **Declared.** A post already under this artist whose body links to + `patreon.com/` for this exact membership. A creator pointing at + their own Patreon from their own Discord is close to a statement. + +Signal 2 is NOT read from `ExternalLink`, and that correction is worth keeping: +`link_extract.SUPPORTED_HOSTS` is file hosts only (mega/gdrive/mediafire/ +dropbox/pixeldrain) and `host_for()` returns None for patreon.com, so no +`ExternalLink` row is ever written for one. The same trap already caught E5 for +Discord invites. + +## Weights, and what they make impossible + + name 0.65 · declared 0.35, cut at 0.60 + +Chosen so the arithmetic encodes the judgement rather than a code path doing it: + +* an EXACT name match alone (0.65) proposes — same slug on both sides is + strong, and requiring corroboration would mean proposing almost nothing; +* a CONTAINMENT name match alone (0.6 * 0.65 = 0.39) does not — "art" inside + "artgirl" is a coincidence generator, and it needs the declaration; +* the declaration ALONE (0.35) never proposes, at any setting at or above + 0.60 — a creator may link another creator's Patreon, and a link is not a + claim of identity. + +A guard test pins all three against WEIGHTS directly, so they survive a +refactor of the scorer. +""" + +from __future__ import annotations + +import logging +import re + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ( + Artist, + ArtistMembershipSuggestion, + PlatformMembership, + Post, + Source, +) +from ..utils.slug import slugify +from ..utils.text import html_to_plain +from .membership_roster import source_for_membership + +log = logging.getLogger(__name__) + +WEIGHTS = {"name": 0.65, "declared": 0.35} +DEFAULT_THRESHOLD = 0.60 + +NAME_EXACT = 1.0 +# Containment is a hint, not a match: "art" sits inside "artgirl", and slugs +# are short enough that coincidental containment is common. +NAME_CONTAINS = 0.6 +# Below this many characters, containment is noise rather than signal — a +# 3-character slug is inside a great many longer ones. +_MIN_CONTAINMENT_LEN = 5 + +MAX_CANDIDATES = 25 + + +def name_signal(membership: PlatformMembership, artist: Artist) -> float: + """Graded slug agreement between a membership and an artist. + + Both the display name and the vanity are tried, because creators routinely + differ between the two ("Team Melon Collie" vs "MelonCollieStudios") and + either may be the one the operator typed when they created the artist. + """ + artist_slug = slugify(artist.name or "") if artist.name else "" + if not artist_slug or artist_slug == "untitled": + return 0.0 + candidates = { + slugify(v) for v in (membership.display_name, membership.vanity_or_none()) + if v + } + candidates.discard("untitled") + if not candidates: + return 0.0 + if artist_slug in candidates: + return NAME_EXACT + for c in candidates: + if len(c) < _MIN_CONTAINMENT_LEN or len(artist_slug) < _MIN_CONTAINMENT_LEN: + continue + if c in artist_slug or artist_slug in c: + return NAME_CONTAINS + return 0.0 + + +def declared_signal(body: str | None, vanity: str | None) -> float: + """Does this post body point at THIS membership's Patreon page? + + Matched against the RAW body, not the stripped text: these links live in an + anchor's `href`, and `html_to_plain` discards attributes — the same trap + that caught E5's invite detection. The stripped text is checked too, for + bodies that paste the URL as plain text. + """ + if not body or not vanity: + return 0.0 + pattern = re.compile( + r"patreon\.com/(?:c/|cw/|checkout/)?" + re.escape(vanity) + r"\b", re.I + ) + if pattern.search(body): + return 1.0 + return 1.0 if pattern.search(html_to_plain(body) or "") else 0.0 + + +def weighted_score(signals: dict) -> float: + return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) + + +class ArtistMembershipService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _decided(self, membership_id: int) -> set[int]: + """Artists already proposed for this membership, in ANY status. + + Dismissed included: the row is what remembers the rejection, and + re-proposing a rejected pair every scan is what makes a queue ignored. + """ + rows = (await self.session.execute( + select(ArtistMembershipSuggestion.artist_id).where( + ArtistMembershipSuggestion.platform_membership_id == membership_id + ) + )).scalars().all() + return set(rows) + + async def _candidate_artists(self, membership: PlatformMembership) -> list[Artist]: + """Artists that have SOME source but none for this membership's platform. + + A hard filter, not a scored signal. An artist FC already tracks on this + platform needs no suggestion — the link exists — and an artist with no + sources at all is not a creator FC is following through another channel, + which is the whole case this step is about. + """ + # `select(...).exists()` rather than a bare `exists().where(...)`: the + # latter has no FROM to correlate against and does not reliably render. + has_any = select(Source.id).where(Source.artist_id == Artist.id).exists() + has_this = ( + select(Source.id) + .where( + Source.artist_id == Artist.id, + Source.platform == membership.platform, + ) + .exists() + ) + return (await self.session.execute( + select(Artist).where(has_any, ~has_this).limit(MAX_CANDIDATES) + )).scalars().all() + + async def _declared_for(self, artist_id: int, vanity: str | None) -> float: + if not vanity: + return 0.0 + # Bounded scan: the newest posts are where a creator's current links + # live, and an unbounded body scan per (artist, membership) pair would + # be the expensive part of this sweep. + bodies = (await self.session.execute( + select(Post.description) + .where(Post.artist_id == artist_id, Post.description.is_not(None)) + .order_by(func.coalesce(Post.post_date, Post.downloaded_at).desc()) + .limit(50) + )).scalars().all() + for body in bodies: + if declared_signal(body, vanity) > 0: + return 1.0 + return 0.0 + + async def match_membership( + self, membership_id: int, *, threshold: float = DEFAULT_THRESHOLD, + ) -> int: + membership = await self.session.get(PlatformMembership, membership_id) + if membership is None: + return 0 + # The shared identity join (C4), used here as the NEGATIVE check. A + # membership FC already has a source for is tracked — whoever it happens + # to be filed under — and proposing it to some OTHER artist would be + # exactly the wrong link this service exists to avoid making. + # `_candidate_artists` only knows whether a GIVEN artist has a source on + # the platform, which cannot see a source sitting under someone else. + if await source_for_membership(self.session, membership) is not None: + return 0 + already = await self._decided(membership_id) + + made = 0 + for artist in await self._candidate_artists(membership): + if artist.id in already: + continue + signals = { + "name": name_signal(membership, artist), + "declared": await self._declared_for( + artist.id, membership.vanity_or_none() + ), + } + score = weighted_score(signals) + if score < threshold: + continue + self.session.add(ArtistMembershipSuggestion( + platform_membership_id=membership.id, + artist_id=artist.id, + score=score, + signals=signals, + status="pending", + )) + made += 1 + return made + + async def list_pending(self) -> list[dict]: + rows = (await self.session.execute( + select(ArtistMembershipSuggestion, PlatformMembership, Artist) + .join( + PlatformMembership, + PlatformMembership.id + == ArtistMembershipSuggestion.platform_membership_id, + ) + .join(Artist, Artist.id == ArtistMembershipSuggestion.artist_id) + .where(ArtistMembershipSuggestion.status == "pending") + .order_by( + ArtistMembershipSuggestion.score.desc(), + ArtistMembershipSuggestion.id.desc(), + ) + )).all() + return [ + { + "id": s.id, + "score": s.score, + "signals": s.signals, + "artist": {"id": a.id, "name": a.name, "slug": a.slug}, + "membership": { + "id": m.id, + "platform": m.platform, + "display_name": m.display_name, + "url": m.url, + }, + } + for s, m, a in rows + ] + + async def accept(self, suggestion_id: int) -> dict | None: + """Add the missing channel to the artist. NEVER merges two artists. + + Returns the created source's id, or `already_linked` when a source for + that platform appeared between the proposal and the click — which is + not an error, it is the operator having done it by hand. + """ + s = await self.session.get(ArtistMembershipSuggestion, suggestion_id) + if s is None: + return None + membership = await self.session.get(PlatformMembership, s.platform_membership_id) + if membership is None or not membership.url: + return None + + existing = (await self.session.execute( + select(Source.id).where( + Source.artist_id == s.artist_id, + Source.platform == membership.platform, + ) + )).scalars().first() + if existing is not None: + s.status = "linked" + return {"id": s.id, "status": s.status, "already_linked": existing} + + # Through SourceService, NOT a bare Source() insert. It carries the + # platform/URL validation, the duplicate check and the #693 + # backfill-arming that a hand-added source gets — building a second, + # quieter way to create a source is how the two drift until one of them + # is subtly broken (rule 28: repurpose the existing surface). + from .source_service import DuplicateSourceError, SourceService + + try: + record = await SourceService(self.session).create( + artist_id=s.artist_id, + platform=membership.platform, + url=membership.url, + ) + except DuplicateSourceError as exc: + # The same URL already exists for this artist — the operator got + # there first by a different route. Not an error. + s.status = "linked" + return {"id": s.id, "status": s.status, "already_linked": exc.existing_id} + s.status = "linked" + return {"id": s.id, "status": s.status, "source_id": record.id} + + async def dismiss(self, suggestion_id: int) -> dict | None: + s = await self.session.get(ArtistMembershipSuggestion, suggestion_id) + if s is None: + return None + # Kept, not deleted — the row is what remembers the rejection. + s.status = "dismissed" + return {"id": s.id, "status": s.status} + + +async def rescan(session: AsyncSession, *, threshold: float = DEFAULT_THRESHOLD) -> dict: + """Offer every known membership to the artists FC already tracks.""" + ids = (await session.execute(select(PlatformMembership.id))).scalars().all() + svc = ArtistMembershipService(session) + proposed = 0 + for mid in ids: + proposed += await svc.match_membership(mid, threshold=threshold) + log.info( + "artist/membership matcher: scanned %d membership(s), proposed %d pair(s)", + len(ids), proposed, + ) + return {"scanned": len(ids), "proposed": proposed} diff --git a/backend/app/services/db_helpers.py b/backend/app/services/db_helpers.py index 992a76d..0a76379 100644 --- a/backend/app/services/db_helpers.py +++ b/backend/app/services/db_helpers.py @@ -20,6 +20,9 @@ from sqlalchemy import Select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession +from ..models import Source +from .gallery_dl import ErrorType + async def get_or_create[T]( session: AsyncSession, @@ -50,3 +53,31 @@ async def get_or_create[T]( except IntegrityError: await sp.rollback() return (await session.execute(select_stmt)).scalar_one(), False + + +# --- shared Source health predicates ---------------------------------------- +# +# The subscriptions rollup, the front-door status ribbon and the list endpoint +# all have to agree on what "failing" and "no access" MEAN, or the ribbon says +# 3 and the card it links to shows 4. Same reasoning as get_or_create above: +# divergent copies of one predicate are how the drift creeps in. Defined here +# rather than in source_service because scheduler_service needs them too, and +# source_service already imports scheduler_service (the other direction would +# be a cycle). + + +def failing_sources_clause(): + """A source is FAILING when its runs are actually erroring. + + Deliberately not `last_error IS NOT NULL` — a tier-limited source clears + last_error and keeps a chip, and must never be counted as broken. + """ + return Source.consecutive_failures > 0 + + +def no_access_sources_clause(): + """A source we can't see the content of: the walk works, the tier doesn't + grant it (#874 / milestone #387 phase A). Not a failure — kept separate + from failing_sources_clause on purpose, and the two are disjoint because + an informational class only ever rides an otherwise-OK run.""" + return Source.error_type == ErrorType.TIER_LIMITED diff --git a/backend/app/services/discord_grouping.py b/backend/app/services/discord_grouping.py new file mode 100644 index 0000000..f0f2b6a --- /dev/null +++ b/backend/app/services/discord_grouping.py @@ -0,0 +1,655 @@ +"""Discord drop grouping — FC authors the post that Discord never wrote. + +Milestone 388, step E2. + +Discord is a delivery CHANNEL, not a publisher. A creator drops a set of +near-variants — the same piece with different hair colour, accessories, an +outfit swap — across a handful of messages, and today each of those messages +lands as its own `post` row, so chat lines compete with authored work for the +same surface. The fix is not to demote them into a second-class feed; it is to +let FC write the post: one row per DROP, its images the drop's images, its body +the messages' text in arrival order. + +The result is post-shaped by construction, which is the entire reason to +synthesise a `Post` rather than invent a parallel entity — feed, provenance, +translation, attachments and series all keep working on it unchanged. + +## The predicate: three axes, ANDed, and the time one does the real work + +**Similarity alone over-groups, and that is the failure that would make this +useless.** Any two pieces of the same character by the same artist sit close in +SigLIP space; a cosine-only rule collapses a month of one character into a +single "post". What makes a variant set a set is that it was dropped TOGETHER. + + same source AND cosine distance <= threshold AND no gap > window + +Two details in there are load-bearing: + +* **Distance is measured to the group's SEED, never to the previous member.** + Chaining to the previous member lets a group DRIFT: twenty small steps walk + from one piece to a completely different one, each hop individually within + threshold. Anchoring on the seed bounds the whole group to one neighbourhood. +* **The window is measured between CONSECUTIVE messages, not from the first.** + An artist trickling variants out over an evening is one drop; a window + anchored on the first message would cut it in half at an arbitrary point. + +## Why this is a post-import sweep and not part of ingest + +The obvious alternative was to migrate Discord to the native post-first +ingester (#1266) and group at capture time. **That cannot work**, and the +reason is worth recording: the grouping signal is `siglip_embedding`, which is +produced ASYNCHRONOUSLY after import (`tasks/ml.py`, the GPU queue backfill). +At capture time the embedding does not exist yet, so an ingester has nothing to +group on. Grouping is necessarily something that happens once the vectors have +caught up — which also means this sweep must be re-runnable and must simply +skip what it cannot yet place. It does: a post whose image has no embedding is +left alone and picked up on a later run. + +## The honesty rule + +A synthetic post must never pretend an artist authored it. It carries +`synthesized_by`, records what it was built from in `synthesis_details` +(members, count, and the thresholds in force at the time), and leaves its +member posts intact and reachable. Deleting the synthetic post releases the +members back into the feed — one DELETE, no repair step. FC invented this +grouping; the operator has to be able to see that, inspect it, and undo it. +""" + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta + +from sqlalchemy import Select, func, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageProvenance, ImageRecord, MLSettings, Post, Source + +log = logging.getLogger(__name__) + +# The value that lands in `post.synthesized_by`. One grouper today; a second +# would be another value here, which is exactly why the column has no CHECK. +DROP_GROUPER = "discord_drop" + +PLATFORM = "discord" + +# Ceiling on member posts examined per source per run. A first sweep over an +# established library would otherwise pull every Discord message's 1152-float +# vector into memory at once. The sweep is re-runnable and works oldest-first, +# so a backlog simply drains over successive runs rather than needing one +# heroic pass. +MAX_CANDIDATES_PER_SOURCE = 500 + + +@dataclass +class DropGroup: + """One drop: the member posts, in arrival order, that will become a post.""" + + member_ids: list[int] = field(default_factory=list) + seed: list[float] | None = None + last_at: datetime | None = None + + +def cosine_distance(a, b) -> float: + """Cosine distance between two embeddings, in the same units pgvector's + `cosine_distance` operator returns (0 = identical, 1 = orthogonal). + + Computed in Python rather than SQL because the comparison is against a + group seed held in a loop, not against a column — and pure arithmetic keeps + numpy off this path entirely. pgvector may hand back a numpy array or a + list depending on driver version, so both are coerced. + """ + va = [float(x) for x in a] + vb = [float(x) for x in b] + # strict=True: two embeddings of different length is a corrupted row or a + # model swap that skipped the re-embed, and silently truncating to the + # shorter one would score it as a near match. + dot = sum(x * y for x, y in zip(va, vb, strict=True)) + na = math.sqrt(sum(x * x for x in va)) + nb = math.sqrt(sum(y * y for y in vb)) + if na == 0.0 or nb == 0.0: + # A zero vector has no direction, so no meaningful distance. Return the + # maximum so it can never pull anything into a group. + return 1.0 + return 1.0 - (dot / (na * nb)) + + +def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select: + """Ungrouped Discord message-posts, one representative image each, OLDEST + FIRST — which is the order `build_groups` requires. + + DISTINCT ON the post picks the lowest-id embedded image as that post's + representative: a Discord message carrying several attachments is still one + point in the drop, and comparing every attachment would let one incidental + image drag an unrelated message into the group. + + The DISTINCT ON is wrapped in a subquery rather than ordered directly, + because Postgres requires a DISTINCT ON query's ORDER BY to LEAD with the + distinct expression — so the inner query must sort by `post.id`, which is + insertion order and not arrival order at all once a backfill has imported + anything out of sequence. Sorting outside is what makes the caller's LIMIT + take the OLDEST candidates instead of the lowest-numbered ones. + """ + sort_key = func.coalesce(Post.post_date, Post.downloaded_at) + inner = ( + select( + Post.id.label("post_id"), + sort_key.label("occurred_at"), + ImageRecord.siglip_embedding.label("embedding"), + ) + .join(ImageRecord, ImageRecord.primary_post_id == Post.id) + .where( + Post.source_id == source_id, + # Never absorb a post FC wrote, and never re-absorb one already + # taken — both would build groups out of groups. + Post.synthesized_by.is_(None), + Post.absorbed_by_post_id.is_(None), + ImageRecord.siglip_embedding.is_not(None), + sort_key <= not_after, + ) + .distinct(Post.id) + .order_by(Post.id, ImageRecord.id) + .subquery() + ) + return ( + select(inner.c.post_id, inner.c.occurred_at, inner.c.embedding) + .order_by(inner.c.occurred_at, inner.c.post_id) + ) + + +def build_groups( + rows: list[tuple[int, datetime, list[float]]], + *, + max_distance: float, + window: timedelta, +) -> list[DropGroup]: + """Walk candidates in arrival order and cut them into drops. + + `rows` must be sorted oldest-first — the whole predicate is about + adjacency in time, so an unsorted input would silently produce nonsense + rather than fail. + """ + groups: list[DropGroup] = [] + current: DropGroup | None = None + + for post_id, occurred_at, embedding in rows: + if current is not None: + gap_ok = occurred_at - current.last_at <= window + # Distance to the SEED, not to the previous member — see the module + # docstring on drift. + near = cosine_distance(current.seed, embedding) <= max_distance + if gap_ok and near: + current.member_ids.append(post_id) + current.last_at = occurred_at + continue + groups.append(current) + current = DropGroup( + member_ids=[post_id], seed=embedding, last_at=occurred_at, + ) + + if current is not None: + groups.append(current) + return groups + + +async def _synthesize( + session: AsyncSession, + *, + source: Source, + group: DropGroup, + max_distance: float, + window_minutes: float, +) -> Post | None: + """Write one synthetic post for `group` and absorb its members.""" + members = (await session.execute( + select(Post) + .where(Post.id.in_(group.member_ids)) + .order_by(func.coalesce(Post.post_date, Post.downloaded_at), Post.id) + )).scalars().all() + if not members: + return None + + first = members[0] + # Deterministic key, so a re-run cannot mint a second post for the same + # drop: the unique (source_id, external_post_id) constraint would reject it + # even if the member filter somehow let the drop through twice. + external_id = f"fc-drop:{first.external_post_id}"[:128] + + # The messages' own text, in arrival order, IS the post's body — that is + # what the operator asked for and it is the only text a drop has. Blank + # messages (an attachment with no caption) contribute nothing rather than a + # run of empty lines. + body = "\n\n".join(m.description.strip() for m in members if m.description and m.description.strip()) + + post = Post( + source_id=source.id, + artist_id=source.artist_id, + external_post_id=external_id, + # post_title stays NULL DELIBERATELY. A synthesised title is the one + # place this feature could accidentally put words in a creator's mouth; + # the UI labels the row from `synthesized_by` instead, which cannot be + # mistaken for something the artist wrote. + post_title=None, + post_url=first.post_url, + post_date=first.post_date or first.downloaded_at, + description=body or None, + synthesized_by=DROP_GROUPER, + synthesis_details={ + "member_post_ids": [m.id for m in members], + "message_count": len(members), + # The thresholds AS THEY WERE. They are operator-tunable, so + # without this "why did it group these" is unanswerable later. + "max_distance": max_distance, + "window_minutes": window_minutes, + "grouped_at": datetime.now(UTC).isoformat(), + # Growth accumulated since the post last moved in the feed (E3). + # Seeded here so the joiner never meets its absence — creation IS + # a surfacing, so the count starts at zero. + "images_since_surface": 0, + }, + ) + session.add(post) + await session.flush() + + await session.execute( + update(Post) + .where(Post.id.in_([m.id for m in members])) + .values(absorbed_by_post_id=post.id) + ) + + await _link_member_images( + session, post_id=post.id, member_ids=[m.id for m in members], + source_id=source.id, + ) + return post + + +async def _link_member_images( + session: AsyncSession, *, post_id: int, member_ids: list[int], source_id: int | None, +) -> int: + """Attach every member's images to the synthetic post. Returns how many. + + The feed and detail views already union provenance with `primary_post_id` + (post_feed_service._thumbnails_for), so this alone makes the drop's images + show up under the post FC wrote — no second render path. + + `primary_post_id` is deliberately NOT rewritten: the message post remains + the image's true origin, and the synthetic post is an ADDITIONAL claim on + it, which is what keeps the grouping reversible. + + Shared by creation and by the E3 joiner rather than written twice, because + the two would otherwise be free to drift on exactly the detail (which post + owns the image) that makes a grouping reversible. + """ + if not member_ids: + return 0 + image_rows = (await session.execute( + select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids)) + )).scalars().all() + if not image_rows: + return 0 + await session.execute( + pg_insert(ImageProvenance) + .values([ + {"image_record_id": iid, "post_id": post_id, "source_id": source_id} + for iid in image_rows + ]) + # (image, post) is unique. This is a BACKSTOP against a re-run that + # raced itself, not the correctness argument: callers only ever pass + # members that were unabsorbed a moment ago, so a conflict here means + # concurrency, not a logic error. + .on_conflict_do_nothing(constraint="uq_image_provenance_image_post") + ) + return len(image_rows) + + +async def group_source( + session: AsyncSession, + source: Source, + *, + max_distance: float, + window_minutes: float, + now: datetime | None = None, +) -> int: + """Group one Discord source's ungrouped messages. Returns posts created.""" + window = timedelta(minutes=window_minutes) + now = now or datetime.now(UTC) + # Leave the most recent window alone: a drop that is still arriving would + # otherwise be cut in half by whichever sweep happened to land mid-drop, + # and the second half would become a separate post claiming to be its own + # drop. Waiting one window costs nothing (the sweep re-runs) and is the E2 + # side of "keep the grouping open"; E3 handles the harder case where a + # matching drop resumes after the gap has already passed. + rows = (await session.execute( + _candidate_stmt(source.id, not_after=now - window) + .limit(MAX_CANDIDATES_PER_SOURCE) + )).all() + if not rows: + return 0 + + groups = build_groups( + [(pid, occurred, emb) for pid, occurred, emb in rows], + max_distance=max_distance, window=window, + ) + if len(rows) == MAX_CANDIDATES_PER_SOURCE and len(groups) > 1: + # The cap may have fallen INSIDE the last drop, and synthesising a + # truncated group would publish a post that claims to be the whole drop + # while the rest of it sits one row past the limit. Leave it for the + # next run, which starts from the same place and sees the remainder. + # Guarded on len > 1 so a single oversized group is not dropped + # forever — it would make no progress at all. + groups = groups[:-1] + + created = 0 + for group in groups: + post = await _synthesize( + session, source=source, group=group, + max_distance=max_distance, window_minutes=window_minutes, + ) + if post is not None: + created += 1 + return created + + +# --------------------------------------------------------------------------- +# E3: an open grouping — a later drop joins its post and updates it. +# --------------------------------------------------------------------------- +# +# A synthetic post is not sealed at creation. A creator who adds two more +# variants the next day extends the existing post rather than starting a new +# one, and its body grows with the new messages. That is what makes chat +# capture read as content TRICKLING IN rather than as a stream of separate +# arrivals. +# +# Three hard problems, each answered deliberately below: bridging (a candidate +# near two groups), re-surfacing without thrashing the feed, and groups that +# stay open forever. + +# How much closer the nearest group must be than the runner-up before a +# candidate is assigned to it at all. +# +# NOT a setting, deliberately. It is not a quality dial the operator would tune +# toward a better feed — it expresses "these two are too close to call", and +# exposing it would invite turning it to zero, which is precisely the silent +# arbitrary choice it exists to prevent. When a candidate is genuinely between +# two groups the recoverable answer is to leave it out and let it start its +# own; the unrecoverable one is to merge, because a merge rewrites history — +# two posts the operator may already have seen become one, and anything +# pointing at the absorbed post dangles. +AMBIGUITY_MARGIN = 0.02 + + +def should_resurface( + *, + images_since_surface: int, + last_surface_at: datetime, + now: datetime, + min_images: int, + cooldown: timedelta, +) -> bool: + """Has this group grown enough, and waited long enough, to move in the feed? + + An updated post SHOULD be visible — that is the point of keeping it open — + but a group gaining one image a day must not sit permanently at the top. + Both conditions have to hold: enough new images that the update is worth an + interruption, and enough time since the last one that a steady drip cannot + chain bumps together. + """ + if images_since_surface < min_images: + return False + return now - last_surface_at >= cooldown + + +async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None: + """The embedding a group is measured against — its FIRST member's image. + + Derived rather than stored, and derived by the same definition `build_groups` + used (earliest member, lowest-id embedded image). Storing it at creation + would have meant a backfill for groups already written and two definitions + free to disagree; this way there is one. + """ + sort_key = func.coalesce(Post.post_date, Post.downloaded_at) + return (await session.execute( + select(ImageRecord.siglip_embedding) + .join(Post, ImageRecord.primary_post_id == Post.id) + .where( + Post.absorbed_by_post_id == post_id, + ImageRecord.siglip_embedding.is_not(None), + ) + .order_by(sort_key, Post.id, ImageRecord.id) + .limit(1) + )).scalar_one_or_none() + + +async def open_groups( + session: AsyncSession, source_id: int, *, now: datetime, close_after: timedelta, +) -> list[tuple[Post, list[float]]]: + """This source's synthetic posts that are still accepting members. + + Openness is DERIVED, not stored: a group is open if it grew — or started — + within `close_after`. A group left open forever would eventually absorb + something it shouldn't, because artists reuse characters for years; a + stored `closed_at` would need a sweep to set it and a repair path to ever + change the policy. This way the policy IS the query. + """ + cutoff = now - close_after + posts = (await session.execute( + select(Post).where( + Post.source_id == source_id, + Post.synthesized_by == DROP_GROUPER, + # A synthetic post that was itself absorbed is not a thing today + # (nothing absorbs one), but joining into one would nest groups. + Post.absorbed_by_post_id.is_(None), + func.coalesce(Post.last_grew_at, Post.post_date, Post.downloaded_at) >= cutoff, + ) + )).scalars().all() + + out: list[tuple[Post, list[float]]] = [] + for post in posts: + seed = await _group_seed(session, post.id) + if seed is not None: + out.append((post, seed)) + return out + + +def assign_to_group( + embedding: list[float], + groups: list[tuple[Post, list[float]]], + *, + max_distance: float, +) -> Post | None: + """Pick the one group this image belongs to, or None to leave it alone. + + Returns None in two cases that mean different things and are deliberately + treated the same: nothing is close enough (so E2 will start a new group + from it), or two groups are BOTH close and too near each other to choose + between (so E2 will start a new group from it). The second is the bridging + case, and letting it start its own post is the recoverable failure — + merging two existing posts is not. + """ + # key= on the distance ALONE. Sorting bare tuples falls through to the + # second element when two distances tie, and `Post` has no ordering — so a + # perfectly symmetric bridge (the exact case this function exists for) + # would raise TypeError instead of declining to choose. + scored = sorted( + ((cosine_distance(seed, embedding), post) for post, seed in groups), + key=lambda pair: pair[0], + ) + within = [(d, p) for d, p in scored if d <= max_distance] + if not within: + return None + if len(within) >= 2 and (within[1][0] - within[0][0]) < AMBIGUITY_MARGIN: + return None + return within[0][1] + + +async def _absorb_into( + session: AsyncSession, + *, + group: Post, + member_ids: list[int], + source_id: int | None, + now: datetime, + min_images: int, + cooldown: timedelta, +) -> int: + """Extend an existing synthetic post with new members. Returns images added.""" + members = (await session.execute( + select(Post) + .where(Post.id.in_(member_ids)) + .order_by(func.coalesce(Post.post_date, Post.downloaded_at), Post.id) + )).scalars().all() + if not members: + return 0 + + added_images = await _link_member_images( + session, post_id=group.id, member_ids=[m.id for m in members], + source_id=source_id, + ) + await session.execute( + update(Post) + .where(Post.id.in_([m.id for m in members])) + .values(absorbed_by_post_id=group.id) + ) + + # The new messages' text joins the body, in arrival order, exactly as at + # creation — the post's body is the drop's text and the drop just grew. + new_text = "\n\n".join( + m.description.strip() for m in members if m.description and m.description.strip() + ) + if new_text: + group.description = f"{group.description}\n\n{new_text}" if group.description else new_text + + details = dict(group.synthesis_details or {}) + existing_ids = list(details.get("member_post_ids") or []) + details["member_post_ids"] = existing_ids + [ + m.id for m in members if m.id not in existing_ids + ] + details["message_count"] = len(details["member_post_ids"]) + since = int(details.get("images_since_surface") or 0) + added_images + details["last_grew_at"] = now.isoformat() + + # The feed position moves only when the anti-thrash rule fires. Measured + # from the last time the post actually MOVED (resurfaced_at), falling back + # to when the drop started — creation is itself a surfacing. + last_surface = group.resurfaced_at or group.post_date or group.downloaded_at + if should_resurface( + images_since_surface=since, last_surface_at=last_surface, now=now, + min_images=min_images, cooldown=cooldown, + ): + group.resurfaced_at = now + since = 0 + details["images_since_surface"] = since + + group.synthesis_details = details + group.last_grew_at = now + return added_images + + +async def join_open_groups( + session: AsyncSession, + source: Source, + *, + max_distance: float, + window_minutes: float, + close_after_hours: float, + resurface_min_images: int, + resurface_cooldown_hours: float, + now: datetime | None = None, +) -> int: + """Offer this source's ungrouped messages to its open groups. + + Runs BEFORE `group_source` in the sweep: a message that belongs to an + existing drop must join it rather than found a rival post, and whichever + runs first wins that message. + """ + now = now or datetime.now(UTC) + window = timedelta(minutes=window_minutes) + groups = await open_groups( + session, source.id, now=now, close_after=timedelta(hours=close_after_hours), + ) + if not groups: + return 0 + + # Same quarantine as E2: a drop still arriving is left for the next run. + rows = (await session.execute( + _candidate_stmt(source.id, not_after=now - window) + .limit(MAX_CANDIDATES_PER_SOURCE) + )).all() + if not rows: + return 0 + + claimed: dict[int, list[int]] = {} + for post_id, _occurred_at, embedding in rows: + target = assign_to_group(embedding, groups, max_distance=max_distance) + if target is not None: + claimed.setdefault(target.id, []).append(post_id) + + by_id = {post.id: post for post, _seed in groups} + joined = 0 + for group_id, member_ids in claimed.items(): + joined += await _absorb_into( + session, group=by_id[group_id], member_ids=member_ids, + source_id=source.id, now=now, + min_images=resurface_min_images, + cooldown=timedelta(hours=resurface_cooldown_hours), + ) + return joined + + +async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict: + """Group every enabled Discord source. No-op when the switch is off. + + Two passes per source, and the ORDER is load-bearing: offer new messages to + the groups that are still open (E3) BEFORE founding new ones (E2). Whichever + runs first claims a message, and a variant that belongs to yesterday's drop + must extend that post rather than found a rival to it. + """ + settings = await MLSettings.load(session) + if not settings.discord_grouping_enabled: + return { + "enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0, + } + + sources = (await session.execute( + select(Source).where( + Source.platform == PLATFORM, + Source.enabled.is_(True), + ) + )).scalars().all() + + max_distance = float(settings.discord_group_max_distance) + window_minutes = float(settings.discord_group_window_minutes) + + created = 0 + joined = 0 + for source in sources: + joined += await join_open_groups( + session, source, + max_distance=max_distance, + window_minutes=window_minutes, + close_after_hours=float(settings.discord_group_close_after_hours), + resurface_min_images=int(settings.discord_group_resurface_min_images), + resurface_cooldown_hours=float( + settings.discord_group_resurface_cooldown_hours + ), + now=now, + ) + created += await group_source( + session, source, + max_distance=max_distance, + window_minutes=window_minutes, + now=now, + ) + log.info( + "discord drop grouping: %d source(s), %d synthetic post(s) created, " + "%d image(s) joined to open groups", + len(sources), created, joined, + ) + return { + "enabled": True, "sources": len(sources), + "posts_created": created, "images_joined": joined, + } diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 49c510a..6d34c07 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -28,12 +28,32 @@ from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .pixiv_client import user_id_from_url from .pixiv_ingester import PixivIngester +from .platforms import known_platform_keys from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until -# they migrate too. -NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"}) +# they migrate too. pixiv left this set when it was retired (milestone #406). +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) + + +def _unsupported_platform_message(platform: str) -> str | None: + """Why `platform` may not be downloaded or verified, or None if it may. + + A source can outlive its platform. Retiring one (DeviantArt #3069, pixiv + #406) unregisters it, but its `Source` rows — and the `enabled` flag on + them — are data, and data survives a deploy. So this refuses at the two + functions every download and every credential probe pass through, instead + of trusting the scheduler's `enabled` filter and every future caller to + agree. + + Without it a retired platform does not fail: it falls through to the + gallery-dl branch, which is precisely where a platform lands once it is no + longer native — and gallery-dl still has an extractor for it. + """ + if platform in known_platform_keys(): + return None + return f"{platform!r} is not a supported platform (retired or unknown)" # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. @@ -80,6 +100,13 @@ async def run_download( backfill state machine and owns phase 3. """ platform = ctx["platform"] + refusal = _unsupported_platform_message(platform) + if refusal is not None: + return DownloadResult( + success=False, url=ctx["url"], artist_slug=ctx["artist_slug"], + platform=platform, + error_type=ErrorType.UNSUPPORTED_URL, error_message=refusal, + ), None if uses_native_ingester(platform): return await _run_native_ingester( ctx, source_config, mode, gdl, sync_session_factory @@ -217,6 +244,11 @@ async def verify_source_credential( network / nothing to test). Callers don't branch on platform — they call this and render the result. """ + refusal = _unsupported_platform_message(platform) + if refusal is not None: + # Inconclusive rather than False: nothing was probed, so nothing was + # rejected. False would tell the operator their credential is bad. + return None, refusal if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe. # SubscribeStar's probe takes the creator URL directly; Patreon's diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 08ca44e..67a5690 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -34,7 +34,9 @@ from .gallery_dl import ( GalleryDLService, SourceConfig, extract_errors_warnings, + is_informational, truncate_log, + walk_completed, ) from .importer import Importer from .platforms import auth_type_for @@ -552,11 +554,13 @@ class DownloadService: # page is no longer double-counted. new_overrides (read fresh above) # carries the ingester's committed value forward untouched. - completed = ( - dl_result.success - and dl_result.error_type is None - and dl_result.return_code == 0 - ) + # Shared with the result path so the two halves can't disagree about + # what "finished" means. Note it admits an INFORMATIONAL error_type: a + # fully-paywalled creator's backfill really did reach the bottom, and + # treating it as unfinished would re-walk that wall every chunk until + # the stall counter tripped — the creator we can see least becoming the + # one we fetch most. + completed = walk_completed(dl_result) if completed: new_overrides["_backfill_state"] = "complete" new_overrides.pop("_backfill_cursor", None) @@ -625,8 +629,17 @@ class DownloadService: if status == "ok": source.consecutive_failures = 0 source.last_error = None - # alembic 0032 — clear the failure-class chip on success. - source.error_type = None + # alembic 0032 — clear the failure-class chip on success, EXCEPT an + # informational class. tier_limited rides an otherwise-successful + # run: failures stay 0 and last_error stays clear (the run did not + # fail and must not earn a backoff), but "there is content here we + # aren't allowed to see" is a durable fact about the SOURCE, not + # about this run. Clearing it here is what left FailingSourcesCard's + # `tier_limited` palette entry unreachable — the chip was wiped by + # the very success that produced it. + source.error_type = ( + error_type if is_informational(error_type) else None + ) elif status == "error": source.consecutive_failures = (source.consecutive_failures or 0) + 1 source.last_error = error_message diff --git a/backend/app/services/extension_service.py b/backend/app/services/extension_service.py index 7a629ab..4b437e4 100644 --- a/backend/app/services/extension_service.py +++ b/backend/app/services/extension_service.py @@ -55,10 +55,6 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P[^/?#]+)", re.IGNORECASE, )), - ("pixiv", re.compile( - r"^https?://(?:www\.)?pixiv\.net/(?:en/)?users/(?P\d+)", - re.IGNORECASE, - )), ] diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 78abd9b..ddaf9f7 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -261,6 +261,72 @@ def make_run_stats( } +# --- tier-gated classification, shared by BOTH backends --------------------- +# +# These three live together because the native ingester and the gallery-dl +# subprocess must reach the same verdict from the same number. They did not: +# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and +# threw the count away, so the platforms FC owns reported a paywalled creator as +# a silent one (#874 follow-up). One predicate, spread into both, rather than +# the condition re-derived per backend. + + +def classify_tier_gated(tier_gated_count: int) -> ErrorType | None: + """TIER_LIMITED when a walk saw tier-gated posts and nothing else failed. + + Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier + posts we cannot see is tier-limited even in a week we did get their cheaper + ones — the fact the operator needs ("there is content here you are not + paying for") is true either way. gallery-dl has classified it this way since + the paywall-as-"needs attention" complaint (see `_categorize_error`), and the + native path now matches rather than inventing a stricter rule. + + Callers must apply this only AFTER the real error categories (auth, rate + limit, drift, …) have had their turn; tier-gating is the weakest signal and + must never mask a genuine failure. + """ + return ErrorType.TIER_LIMITED if tier_gated_count else None + + +def tier_gated_message(count: int) -> str: + """The one wording for the tier-gated verdict, so the two backends can't + describe the same state differently in the Logs UI.""" + return ( + f"Subscription tier does not grant access to " + f"{count} post{'s' if count != 1 else ''}" + ) + + +# `Source.error_type` doubles as the failure-class chip, and a status of "ok" +# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an +# otherwise-successful run, so without an exemption the chip is wiped the moment +# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never +# render. Informational classes are the exemption — they describe the source, +# not a failure of the run. +INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value}) + + +def is_informational(error_type) -> bool: + """True for a class that reports a state rather than a failure. Accepts an + ErrorType or the plain string persisted on Source.error_type.""" + return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES + + +def walk_completed(result: DownloadResult) -> bool: + """Did this walk reach the bottom cleanly? + + The backfill lifecycle's completion test. An informational error_type still + counts as complete: a fully-paywalled creator's backfill DID finish, and + treating it as unfinished re-walks the same wall until the stall counter + trips — the creator we can see least becoming the one we fetch most. + """ + return ( + result.success + and result.return_code == 0 + and (result.error_type is None or is_informational(result.error_type)) + ) + + class GalleryDLService: """Service for executing gallery-dl downloads.""" @@ -531,12 +597,12 @@ class GalleryDLService: line for line in combined.split("\n") if "][warning]" in line and "not allowed to view post" in line ] - if tier_gated_lines: - count = len(tier_gated_lines) - return ( - ErrorType.TIER_LIMITED, - f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", - ) + # Same predicate + wording the native path uses, so the two backends + # can't drift on what counts as tier-gated or how it reads. + count = len(tier_gated_lines) + gated = classify_tier_gated(count) + if gated is not None: + return (gated, tier_gated_message(count)) # Partial-success: the subprocess exited non-zero (typically because # the wall-clock timeout fired mid-walk), but it had downloaded ≥1 diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 38f7eb7..ead2f1e 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -35,7 +35,13 @@ from collections.abc import Callable from sqlalchemy import delete, func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert -from .gallery_dl import DownloadResult, ErrorType, make_run_stats +from .gallery_dl import ( + DownloadResult, + ErrorType, + classify_tier_gated, + make_run_stats, + tier_gated_message, +) from .native_ingest_common import NativeAuthError, NativeDriftError log = logging.getLogger(__name__) @@ -245,6 +251,12 @@ class Ingester: per_item_failures=errors, quarantined_count=quarantined, dead_lettered_count=dead_lettered, + # #874 follow-up: the native path counted gated posts but + # never reported them, so DownloadDetailModal's "Tier-gated" + # field read 0 on every native walk while gallery-dl's read + # true. A paywalled creator was indistinguishable from a + # silent one. + tier_gated_count=gated_skipped, ), ) @@ -464,6 +476,10 @@ class Ingester: "errors": errors, "quarantined": quarantined, "posts": posts_processed, + # Ticks during the walk, not only at finalization: a + # deep backfill on a creator we've lost access to is + # otherwise a long run of zeros with no explanation. + "gated": gated_skipped, }) if early_out: @@ -564,17 +580,33 @@ class Ingester: error_type=ErrorType.API_DRIFT, error_message=msg, ) - # Normal success: reached the bottom, or a tick that early-outed. rc 0 + - # error_type None is REQUIRED for a backfill/recovery walk that reached - # the bottom to be marked COMPLETE by - # download_service._apply_backfill_lifecycle — so we return None even - # when downloaded == 0 (a re-confirming walk that found nothing new still - # completed). success=True maps to status "ok" regardless. A tick that - # early-outed also returns here; ticks never set backfill state so the - # lifecycle is a no-op for them. + # Normal success: reached the bottom, or a tick that early-outed. A + # zero-download walk still returns success here — a re-confirming walk + # that found nothing new genuinely completed. A tick that early-outed + # also lands here; ticks never set backfill state so the lifecycle is a + # no-op for them. + # + # success=True and return_code=0 are load-bearing, not cosmetic. They + # are what make this a COMPLETE walk for + # download_service._apply_backfill_lifecycle (via walk_completed) and + # what map it to status "ok", so a walk that fetched nothing doesn't + # accrue consecutive_failures or a backoff it hasn't earned. + # + # #874 follow-up: "nothing new" and "everything sat behind a tier you + # don't hold" are different facts, and returning None for both made a + # paywalled creator indistinguishable from a silent one. TIER_LIMITED is + # classified LAST — every real failure has already returned above — + # because tier-gating is the weakest signal and must never mask a + # genuine error. It is informational, so walk_completed still counts + # this walk as finished (see that predicate for why re-walking a + # paywalled creator forever is the bug being avoided). + gated_error = classify_tier_gated(gated_skipped) return _result( success=True, return_code=0, - error_type=None, error_message=None, + error_type=gated_error, + error_message=( + tier_gated_message(gated_skipped) if gated_error else None + ), ) # -- failure mapping (adapter overrides) ------------------------------- diff --git a/backend/app/services/membership_reconcile.py b/backend/app/services/membership_reconcile.py new file mode 100644 index 0000000..c6a0d44 --- /dev/null +++ b/backend/app/services/membership_reconcile.py @@ -0,0 +1,235 @@ +"""Reconciling the learned roster against the sources FC actually tracks. + +Milestone 387, step C4. The step the operator asked for; C0-C3 are what make it +trustworthy enough to act on. + +## The buckets + +1. `subscribed_not_tracked` — you pay for this and FC does not follow it. The + adoption win, and the only bucket carrying an action. +2. `tracked_not_subscribed` — FC follows this and the roster does not show you + paying for it. REPORT ONLY, by the operator's decision (2026-09-11): it says + what it sees and links to the existing Subscriptions row, and offers no + one-click disable. +3. `matched` — the healthy set. Counted, not listed loudly. +4. `unidentified` — sources this join cannot speak to at all. Reported as + exactly that, because the alternative is filing them under a verdict. + +## Why absence is the dangerous direction + +Bucket 1 is safe to be wrong about: the cost of offering a source the operator +does not want is one ignored row. Bucket 2 is not. It is computed from an +ABSENCE — no membership matched — and three different things produce that +absence: the subscription genuinely lapsed, the sweep failed, or the creator +renamed and this source has never been walked so no exact id was ever cached. + +Two guards follow from that, and they are the substance of this module: + +* the whole bucket is gated on `roster_is_fresh`, so a failed or never-run sweep + yields an empty list rather than a confident accusation (C3 built the state + this reads); +* every row carries the BASIS for its claim, so "your membership says former + patron" and "we know this creator's id and it is not in your roster" and "we + only have a URL handle to go on" are three different sentences rather than one + overconfident one. + +`has_paid_access` returning None is honoured throughout: unknown is never +rendered as lapsed. That is the whole reason it returns a tri-state. +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, MembershipSync, PlatformMembership, Source +from .membership_roster import ( + get_sync_state, + has_paid_access, + identity_keys_for_source, + pair_sources_with_memberships, + roster_is_fresh, + url_tail, +) + +# Why a source appears in `tracked_not_subscribed`. Ordered strongest first — +# the UI renders a different sentence per basis, because collapsing them into +# one would make the weakest claim sound like the strongest. +BASIS_LAPSED = "lapsed" # a matched membership says access ended +BASIS_ABSENT_EXACT = "absent_exact" # exact id known, not in a fresh roster +BASIS_ABSENT_HANDLE = "absent_handle" # only a URL handle to go on + + +def _membership_row(m: PlatformMembership) -> dict: + return { + "id": m.id, + "platform": m.platform, + "external_campaign_id": m.external_campaign_id, + "display_name": m.display_name or m.vanity_or_none(), + "url": m.url, + "vanity": m.vanity_or_none(), + "status": m.status, + "tier_names": m.tier_names, + "amount_cents": m.amount_cents, + "currency": m.currency, + "paid_access": has_paid_access( + m.platform, m.status, + is_free_member=bool((m.details or {}).get("is_free_member")), + ), + } + + +def _source_row(source: Source, artist: Artist) -> dict: + return { + "id": source.id, + "platform": source.platform, + "url": source.url, + "enabled": source.enabled, + "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, + } + + +async def reconcile( + session: AsyncSession, *, platform: str, now: datetime | None = None, +) -> dict: + """Sort one platform's memberships and sources into the four buckets. + + Always returns the COMPLETE shape, including when the roster is not fresh — + a caller reading `len(result["tracked_not_subscribed"])` must not have to + check which keys exist first. `fresh` is what says whether the emptiness + means anything. + """ + state = await get_sync_state(session, platform) + fresh = roster_is_fresh(state, now=now) + + memberships = (await session.execute( + select(PlatformMembership).where(PlatformMembership.platform == platform) + )).scalars().all() + rows = (await session.execute( + select(Source, Artist) + .join(Artist, Artist.id == Source.artist_id) + .where(Source.platform == platform) + )).all() + + # The join itself lives in `membership_roster` beside `match_kind`, so C5's + # gated-reason annotation pairs sources with memberships by exactly the same + # rule this card sorts them by. Two copies would let the Subscriptions row + # and this card disagree about which creator a source IS. + pairs = pair_sources_with_memberships([s for s, _a in rows], memberships) + matched_membership_ids = {m.id for m, _kind in pairs.values()} + + subscribed_not_tracked = [] + for m in memberships: + if m.id in matched_membership_ids: + continue + paid = has_paid_access( + m.platform, m.status, + is_free_member=bool((m.details or {}).get("is_free_member")), + ) + # A membership FC knows has ENDED is not an adoption opportunity — + # adding it would start a walk that can only fetch what is already + # public. Unknown (None) is still offered: the operator can judge it, + # and refusing to show it would hide a real subscription behind a word + # this code has not been taught. + if paid is False: + continue + subscribed_not_tracked.append(_membership_row(m)) + + tracked_not_subscribed = [] + matched = [] + unidentified = [] + for source, artist in rows: + pair = pairs.get(source.id) + if pair is not None: + m, kind = pair + paid = has_paid_access( + m.platform, m.status, + is_free_member=bool((m.details or {}).get("is_free_member")), + ) + if paid is False: + if not source.enabled: + # Already off. Reporting a source the operator has already + # stopped following is noise, not a finding. + continue + row = _source_row(source, artist) + row["basis"] = BASIS_LAPSED + row["matched_by"] = kind + row["membership"] = _membership_row(m) + tracked_not_subscribed.append(row) + else: + row = _source_row(source, artist) + row["matched_by"] = kind + row["membership"] = _membership_row(m) + matched.append(row) + continue + + # No membership matched. Whether that MEANS anything depends entirely on + # how well this source can be identified at all. + has_exact = bool(identity_keys_for_source(source)) + if not has_exact and url_tail(source.url) is None: + # Nothing to match on — a sidecar anchor or a URL with no handle. + # Reported as unidentified rather than silently dropped, so the + # counts add up to the source list the operator can see. + unidentified.append(_source_row(source, artist)) + continue + if not source.enabled: + # Already off. Telling the operator to stop following something they + # have stopped following is noise, not a finding. + continue + row = _source_row(source, artist) + row["basis"] = BASIS_ABSENT_EXACT if has_exact else BASIS_ABSENT_HANDLE + row["matched_by"] = None + row["membership"] = None + tracked_not_subscribed.append(row) + + # THE GATE. Everything above computed the bucket; this decides whether it may + # be shown. A stale or never-run roster makes every absence meaningless, and + # an absence rendered as a verdict is how this feature would tell the + # operator to cancel something they are still paying for. + if not fresh: + tracked_not_subscribed = [] + + return { + "platform": platform, + "fresh": fresh, + # How many sources exist on this platform at all. The UI needs it to + # decide whether an untrustworthy roster is worth mentioning: with no + # sources here there is nothing to reconcile, and a stale-roster warning + # would be noise on an install that simply has not started yet (that + # empty-install case is C6's, not this card's). + "tracked_total": len(rows), + "last_success_at": ( + state.last_success_at.isoformat() + if state is not None and state.last_success_at else None + ), + "subscribed_not_tracked": subscribed_not_tracked, + "tracked_not_subscribed": tracked_not_subscribed, + "matched": matched, + "unidentified": unidentified, + } + + +async def reconcile_all(session: AsyncSession, now: datetime | None = None) -> dict: + """Every platform the roster knows about, in one payload for the UI. + + The platform list is the UNION of platforms with memberships and platforms + with sync state, not just the former. A sweep that has never succeeded has + recorded zero memberships, and deriving the list from memberships alone + would drop exactly that platform from the payload — making a broken + credential indistinguishable from a platform FC was never asked about. That + distinction is the whole reason C3 records sync state. + """ + with_memberships = (await session.execute( + select(PlatformMembership.platform).distinct() + )).scalars().all() + with_state = (await session.execute( + select(MembershipSync.platform) + )).scalars().all() + platforms = set(with_memberships) | set(with_state) + return { + "platforms": [ + await reconcile(session, platform=p, now=now) for p in sorted(platforms) + ] + } diff --git a/backend/app/services/membership_roster.py b/backend/app/services/membership_roster.py new file mode 100644 index 0000000..8cf4945 --- /dev/null +++ b/backend/app/services/membership_roster.py @@ -0,0 +1,519 @@ +"""The learned membership roster: what the account actually subscribes to. + +Milestone 387, phase C. Sibling of `service_roster` (milestone 365) and built +on the same insight — an absence is only observable against a record of +presence. There, a stopped worker; here, a subscription that lapsed. + +## Nothing calls this yet + +`touch_membership` is written before its caller because the caller (the sweep, +C3) needs a client seam (C2) that needs Patreon's real response characterised +from a captured sample (C0), and that capture needs the operator's browser +session. The write side does not depend on any of it: an upsert keyed on +(platform, external_campaign_id) is the same regardless of what the payload +turns out to look like, and `details` carries whatever C0 finds. + +## Why the whitelist lives here and not in the column + +`platform_membership.status` is an unconstrained String holding the PLATFORM's +own word — `active_patron`, not some normalised FC value. The mapping from +those words to FC's meaning is a read-site concern and belongs in code that can +be corrected without a migration, because the vocabulary comes from whatever +each platform says and will be discovered per platform rather than designed up +front. `MEMBERSHIP_STATUS` below is a place for that knowledge to accumulate as +platforms are characterised; it is deliberately empty of guesses today. +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime, timedelta + +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import MembershipSync, PlatformMembership, Source + +log = logging.getLogger(__name__) + +# Platform word -> whether the account currently has paid access. +# +# Every entry here must come from a CHARACTERISED response, never from API docs +# or a plausible guess — project rule 130, and inventing a status before seeing +# it in a real payload is exactly the failure it names. +# +# patreon: from a live capture of the operator's own session, 2026-09-10 +# (Scribe note #3886). Only two values were OBSERVED in `patron_status` and +# only those two are here. +# +# `declined_patron` is deliberately ABSENT even though it looks obviously +# right. It appears in the request's `filter[membership_type]`, and the capture +# proved that filter is NOT the same vocabulary as the attribute — a row +# selected by the filter as `free_member` came back with +# `patron_status: former_patron`, a word the filter does not contain. Reading +# the filter as an enum is the specific mistake the capture caught; adding +# `declined_patron` on the strength of it would be repeating that mistake one +# step later. +# +# Unknown words are NOT an error: an unrecognised status means the roster +# records evidence it cannot yet interpret, which is a better state than +# dropping the row or asserting a meaning for it. +# +# subscribestar: from a live capture of the account's /subscriptions page, +# 2026-09-13 (Scribe note #3989). SubscribeStar gives NO per-row status word — +# a membership's state is which of two tables it sits in — so the "word" stored +# is the table card's own `data-identifier`, verbatim. Those two identifiers are +# the whole vocabulary; there is nothing further to characterise later. +MEMBERSHIP_STATUS: dict[str, dict[str, bool]] = { + "patreon": { + "active_patron": True, + "former_patron": False, + }, + "subscribestar": { + "active_subscriptions": True, + "cancelled_subscriptions": False, + }, +} + + +def has_paid_access( + platform: str, status: str | None, *, is_free_member: bool = False, +) -> bool | None: + """Does this membership mean the account currently PAYS for access? + + Returns None for a status this code has not been taught, which callers must + treat as "unknown" rather than as False. The difference matters: False says + the operator has lost access, and asserting that from an unrecognised word + would tell them to cancel a source they are still paying for. + + `is_free_member` is a second axis, not a status, and that is Patreon's + design rather than ours: the capture shows a free follow expressed as a + boolean alongside `patron_status`, so a "current" membership can still be + one nobody is paying for. Taking status alone would report a free follower + as a paying patron, and C4 would then never offer to clean it up. + + (Honest limit: the capture contains no ACTIVE free member, so it cannot + demonstrate the two axes coming apart. The separation is what the payload's + shape says; the sample only shows it is possible, not that it happens.) + """ + if status is None: + return None + known = MEMBERSHIP_STATUS.get(platform, {}).get(status) + if known is None: + return None + if not known: + return False + return not is_free_member + + +async def touch_membership( + session: AsyncSession, + *, + platform: str, + external_campaign_id: str, + display_name: str | None = None, + url: str | None = None, + status: str | None = None, + tier_names: list | None = None, + amount_cents: int | None = None, + currency: str | None = None, + details: dict | None = None, +) -> None: + """Record that this membership was observed just now. + + Upsert rather than read-modify-write, for the same reason as + `service_roster.touch_service`: a sweep may overlap its own previous run, + and the last writer is simply the most recent sighting. + + `first_seen_at` is deliberately NOT in the update set. It is the one field + that answers "has this ever been true", which is what makes a membership's + later DISAPPEARANCE readable as a lapse rather than indistinguishable from + a creator FC never knew about. Every other column is last-writer-wins, + including status — a membership that goes from active to former must move. + """ + stmt = pg_insert(PlatformMembership).values( + platform=platform, + external_campaign_id=external_campaign_id, + display_name=display_name, + url=url, + status=status, + tier_names=tier_names, + amount_cents=amount_cents, + currency=currency, + details=details or {}, + ) + stmt = stmt.on_conflict_do_update( + constraint="uq_platform_membership_platform_campaign", + set_={ + "display_name": stmt.excluded.display_name, + "url": stmt.excluded.url, + "status": stmt.excluded.status, + "tier_names": stmt.excluded.tier_names, + "amount_cents": stmt.excluded.amount_cents, + "currency": stmt.excluded.currency, + "details": stmt.excluded.details, + "last_seen_at": func.now(), + }, + ) + await session.execute(stmt) + + +# --------------------------------------------------------------------------- +# The sweep, and the state that makes its failures readable (#387 C3) +# --------------------------------------------------------------------------- +# +# How long a successful sync stays trustworthy. Beyond this the roster is +# STALE, and C4 must refuse to draw conclusions from it — "you are tracking 12 +# sources you no longer subscribe to", computed from a roster that stopped +# syncing a week ago, is an invitation to cancel things the operator is still +# paying for. +# +# Generous relative to the daily cadence: a few missed runs are a blip, not a +# reason to stop trusting a roster that changes on a billing cycle. +ROSTER_STALE_AFTER = timedelta(days=3) + + +async def get_sync_state(session: AsyncSession, platform: str) -> MembershipSync | None: + return (await session.execute( + select(MembershipSync).where(MembershipSync.platform == platform) + )).scalar_one_or_none() + + +def roster_is_fresh(state: MembershipSync | None, *, now: datetime | None = None) -> bool: + """May a caller draw CONCLUSIONS from this roster? + + False for never-synced and for stale, and those are deliberately the same + answer here even though the UI must tell them apart: both mean the roster + is not evidence. The asymmetry that matters is that `False` never means + "you subscribe to nothing" — it means "we do not know", and a caller that + cannot represent "we do not know" must not be asking this question. + """ + if state is None or state.last_success_at is None: + return False + now = now or datetime.now(UTC) + return (now - state.last_success_at) <= ROSTER_STALE_AFTER + + +async def _record_sync(session: AsyncSession, platform: str, **values) -> None: + stmt = pg_insert(MembershipSync).values(platform=platform, **values) + await session.execute(stmt.on_conflict_do_update( + constraint="uq_membership_sync_platform", + set_={**values, "updated_at": func.now()}, + )) + + +def roster_user_id(client) -> str | None: + """The account id a client's roster walk needs, if that client needs one. + + Patreon's members endpoint filters on the account's own user id, so the + sweep has to resolve it first. SubscribeStar's /subscriptions page is simply + the logged-in account's, with nothing to resolve. Probed with `getattr`, + the same way the sweep probes `iter_memberships` itself (rule #169), rather + than called unconditionally. + + Calling `current_user_id()` unconditionally was the one place the membership + seam was still Patreon-shaped: note #3970 promised a second platform would be + one `builders` line plus the client method, and D1 found the sweep would + instead have crashed on the first client without that method. + """ + resolve = getattr(client, "current_user_id", None) + return resolve() if resolve is not None else None + + +async def sync_platform( + session: AsyncSession, + *, + platform: str, + fetch: Callable[[], Awaitable[list]], + now: datetime | None = None, +) -> dict: + """Walk one platform's roster and record what happened. + + `fetch` is injected rather than built here so the error-to-state mapping — + the part with the consequences — is testable without a credential, and so + this service needs to know nothing about how any particular client is + constructed. + + THE FETCH COMPLETES BEFORE ANYTHING IS WRITTEN. That ordering is the whole + safety property: a walk that dies half way through pagination writes + nothing, so a failure can never leave a roster that is partly this week's + and partly last week's. (`touch_membership` never deletes, so a failure + cannot empty the roster either — but "intact" should mean intact, not + merely non-empty.) + + Returns a summary dict; never raises for a platform failure, because one + platform failing must not abort the others. + """ + now = now or datetime.now(UTC) + await _record_sync(session, platform, last_attempt_at=now) + await session.commit() + + try: + memberships = await fetch() + except Exception as exc: # noqa: BLE001 - deliberately broad, see below + # Broad on purpose: a sweep is a background job, and ANY escape here + # kills the run for every other platform too. The exception's class + # name is recorded so the distinction the client drew (auth vs drift + # vs transport) survives into the UI, which is where it is actionable. + # + # EXCEPT the worker asking us to stop. Celery raises its soft time + # limit as an ordinary Exception subclass, so a broad catch swallows + # the shutdown request and lets the sweep run on into the HARD limit, + # where it is SIGKILLed mid-transaction. A sweep that cannot be stopped + # is worse than one that fails. (KeyboardInterrupt and SystemExit are + # BaseException and pass through this clause already.) + from celery.exceptions import SoftTimeLimitExceeded + + if isinstance(exc, SoftTimeLimitExceeded): + raise + await session.rollback() + await _record_sync( + session, platform, + last_error_type=type(exc).__name__, + last_error_message=str(exc)[:2000], + ) + await session.commit() + log.warning("membership sync failed for %s: %s", platform, exc) + return {"platform": platform, "ok": False, "error": type(exc).__name__} + + for m in memberships: + await touch_membership( + session, + platform=platform, + external_campaign_id=m.campaign_id, + display_name=m.display_name, + url=m.url, + status=m.status, + tier_names=m.tier_names or None, + amount_cents=m.amount_cents, + currency=m.currency, + details={**(m.details or {}), "is_free_member": m.is_free_member}, + ) + await _record_sync( + session, platform, + last_success_at=now, + last_count=len(memberships), + # Cleared on success — a stale error beside a fresh success would read + # as "still broken" forever. + last_error_type=None, + last_error_message=None, + ) + await session.commit() + log.info("membership sync ok for %s: %d membership(s)", platform, len(memberships)) + return {"platform": platform, "ok": True, "count": len(memberships)} + + +# --------------------------------------------------------------------------- +# Membership <-> Source identity (#387 C4) +# --------------------------------------------------------------------------- +# +# "Is this membership already tracked?" is asked by TWO features — C4's +# reconciliation buckets and E4's creator suggestions — and it lives here, once, +# on purpose. Built inline in C4 it would have looked finished while leaving E4 +# matching on name similarity alone, so the two would answer the same question +# differently and only one of them would be right. +# +# E4 and C4 use it from opposite sides: E4 as the NEGATIVE check (propose only +# where nothing matches) and C4 as the join itself. + +# Any platform that caches its creator id does so under this suffix; see +# `download_service._phase3_persist`, which writes `patreon_campaign_id`. +_CAMPAIGN_KEY_SUFFIX = "_campaign_id" + + +def identity_keys_for_source(source: Source) -> set[str]: + """Every platform-side creator id cached on this source. + + Reads ANY `_campaign_id` override rather than naming Patreon's, + so a second platform participates by caching its id under the same suffix — + no registry, no `if platform ==` branch (rule 169). A source that has never + been walked has cached nothing and simply contributes no exact key, which is + what makes the handle fallback below necessary rather than merely tolerated. + """ + keys = set() + for name, value in (source.config_overrides or {}).items(): + if name.endswith(_CAMPAIGN_KEY_SUFFIX) and isinstance(value, str) and value: + keys.add(value) + return keys + + +def url_tail(url: str | None) -> str | None: + """The creator handle at the end of a source URL, lowercased. + + Deliberately the same derivation as `PlatformMembership.vanity_or_none`'s + own fallback, so both sides of the comparison reduce a URL to a handle the + same way. Query strings and fragments are stripped first; Patreon's `/c/` + and `/cw/` forms both end in the vanity, so they need no special case (the + missing-`/c/` regex is what broke creator detection in #1485). + + Returns None for the pre-0030 `sidecar:` synthetic anchors, which are not + feeds and must never match anything. + """ + if not url or url.startswith("sidecar:"): + return None + cleaned = url.split("?", 1)[0].split("#", 1)[0] + tail = cleaned.rstrip("/").rsplit("/", 1)[-1] + return tail.lower() or None + + +def match_kind(source: Source, membership: PlatformMembership) -> str | None: + """How this source and this membership are known to be the same creator. + + Returns "campaign" for an exact platform-id match, "vanity" for agreeing URL + handles, or None for no evidence. + + THE ORDER MUST NOT BE INVERTED. The campaign id is exact and the handle is + not, but the id is only written AFTER a source has been walked at least once + — so checking the handle first would let a stale or renamed URL outvote the + authoritative id on every source FC has actually polled. + """ + if source.platform != membership.platform: + return None + if membership.external_campaign_id in identity_keys_for_source(source): + return "campaign" + vanity = membership.vanity_or_none() + tail = url_tail(source.url) + if vanity and tail and vanity.strip().lower() == tail: + return "vanity" + return None + + +def pair_sources_with_memberships( + sources: list[Source], memberships: list[PlatformMembership], +) -> dict[int, tuple[PlatformMembership, str]]: + """Source id -> the membership it is the same creator as, and how we know. + + Extracted from C4's reconcile loop when C5 became its second caller. It is + a nested loop rather than a SQL join because the match is a predicate over + a JSON blob and a derived URL handle, neither of which is indexable, and + both sides are tens of rows on any real library. Keeping it in Python means + ONE definition of identity (`match_kind`) instead of a second one in SQL + that could drift from it. + + First match wins, which is `match_kind`'s ordering doing its job: a source + with a cached campaign id can only pair with the membership holding that + id, so an ambiguous handle never outvotes it. + """ + pairs: dict[int, tuple[PlatformMembership, str]] = {} + for source in sources: + for m in memberships: + kind = match_kind(source, m) + if kind: + pairs[source.id] = (m, kind) + break + return pairs + + +# --------------------------------------------------------------------------- +# Why the posts are invisible (#387 C5) +# --------------------------------------------------------------------------- +# +# A3 made a tier-gated source say "47 posts you can't see". These are the words +# the roster is allowed to add to that count — and ONLY to that count. +# +# THE LINE: the roster ANNOTATES the gated flag, it never produces it. +# `current_user_can_view` (read per post by `patreon_client.post_is_gated`) is +# the authoritative per-post signal, and entitled-tier data cannot stand in for +# it — a creator can gate a post behind an access rule that maps onto no tier +# name at all. So nothing here may suppress a download, skip a walk, or decide +# a post is inaccessible. It explains a skip that ALREADY happened. Getting +# that backwards would make FC silently stop fetching content the operator is +# paying for, which is the worst failure available in this milestone. +# `test_no_fetch_path_can_read_the_roster` pins that structurally. +GATED_LAPSED = "lapsed" # the membership ended — resubscribe, or disable +GATED_TIER = "tier" # paying, but this tier doesn't reach these posts +GATED_FREE = "free" # a current FREE follow — nobody is paying for access + + +def gated_reason( + platform: str, status: str | None, *, is_free_member: bool = False, +) -> str | None: + """Why a tier-gated source's posts are out of reach, if the roster knows. + + None means "no words beyond the count" and is the answer for every case + where the roster is not evidence: a status this code has not been taught, + and (at the call site) a campaign absent from the roster or a roster too + stale to trust. Absence is not evidence — the same discipline as + `test_post_is_gated_only_on_explicit_false`. + + `is_free_member` is read AFTER the status axis, not folded into it, which + is why `has_paid_access` is called here with it forced off. The two axes + are independent in Patreon's payload, and collapsing them loses a real + distinction: a current free follower has not lost anything, so telling them + "you're not a patron any more" would be a false sentence about a state they + were never in. + """ + by_status = has_paid_access(platform, status, is_free_member=False) + if by_status is None: + return None + if not by_status: + return GATED_LAPSED + return GATED_FREE if is_free_member else GATED_TIER + + +async def gated_reasons_for_sources( + session: AsyncSession, sources: list[Source], *, now: datetime | None = None, +) -> dict[int, str]: + """The reason word for each of these sources, where the roster has one. + + Callers pass ONLY the sources already known to be tier-gated: the question + "why can't I see these posts" is meaningless for a source whose posts are + all visible, and asking it anyway would put roster data on rows that have + no gated state for it to annotate. + + Sources with no entry in the result get A3's bare count, which is the + correct degraded rendering for all three of: platform never swept, roster + stale, campaign not in the roster. + """ + if not sources: + return {} + + platforms = {s.platform for s in sources} + # Per platform, because freshness is per platform: a working Patreon sweep + # must not lend its credibility to a SubscribeStar roster that has never + # run. Same gate as C4's `tracked_not_subscribed`, for the same reason. + fresh = { + p for p in platforms + if roster_is_fresh(await get_sync_state(session, p), now=now) + } + if not fresh: + return {} + + memberships = (await session.execute( + select(PlatformMembership).where( + PlatformMembership.platform.in_(sorted(fresh)) + ) + )).scalars().all() + pairs = pair_sources_with_memberships( + [s for s in sources if s.platform in fresh], memberships, + ) + + reasons: dict[int, str] = {} + for source_id, (m, _kind) in pairs.items(): + reason = gated_reason( + m.platform, m.status, + is_free_member=bool((m.details or {}).get("is_free_member")), + ) + if reason is not None: + reasons[source_id] = reason + return reasons + + +async def source_for_membership( + session: AsyncSession, membership: PlatformMembership, +) -> Source | None: + """The source FC already tracks for this membership, if there is one. + + Scoped to the membership's own platform, so a creator tracked on Discord and + subscribed to on Patreon does not read as already-tracked — that pairing is + E4's suggestion to make, not an identity. + """ + rows = (await session.execute( + select(Source).where(Source.platform == membership.platform) + )).scalars().all() + for source in rows: + if match_kind(source, membership): + return source + return None diff --git a/backend/app/services/native_ingest_common.py b/backend/app/services/native_ingest_common.py index e0c9bda..7731163 100644 --- a/backend/app/services/native_ingest_common.py +++ b/backend/app/services/native_ingest_common.py @@ -211,6 +211,59 @@ class PostRecordOutcome: body_chars: int +# -- membership roster seam (shared dataclass, #387 C2/C7) ----------------- + +@dataclass +class Membership: + """One membership the ACCOUNT holds, as the roster needs it (#387 C2). + + Lives HERE rather than in the platform module that first produced it, for + the same reason `PostRecordOutcome` does: it is the seam's contract, not + Patreon's. C7 moved it — while it sat in `patreon_client` a second platform + would have had to import its contract from the first platform's module, + which inverts the dependency and is how a "portable" seam quietly becomes + Patreon-shaped. + + Deliberately not a raw upstream row: the sweep should not have to know that + a tier lives behind a JSON:API `reward` relationship, and + `platform_membership` should not gain columns because one platform shapes + things a certain way. + + `status` carries the PLATFORM's own word, verbatim and unmapped + (`active_patron`, `former_patron`, ...). Deciding what it means is the read + site's job — `membership_roster.has_paid_access` — precisely so an + unrecognised word records as evidence rather than as a decision. + + `is_free_member` is SEPARATE from status and must stay that way. Patreon + expresses a free follow as this boolean rather than as a status value, so + "does the account pay for this" is `status == "active_patron" and not + is_free_member` — a question the status string alone cannot answer. NOTE: + the C0 capture contains no ACTIVE free member, so the two fields are + perfectly correlated in that sample; the separation is what the schema + says, not something the sample proves. + + A platform that lacks a field supplies the empty answer, never a guess: + no tiers -> `[]`, no pledge -> `amount_cents=None` (absent stays + distinguishable from zero — "free" and "we don't know" are different + answers), no vanity -> None and identity falls back to the URL tail. + """ + + campaign_id: str + display_name: str | None + url: str | None + vanity: str | None + status: str | None + is_free_member: bool + tier_names: list[str] + amount_cents: int | None + currency: str | None + # Everything the roster did not model, kept so a later question can be + # answered without another authenticated round-trip. Scoped to the + # membership's own attributes plus the creator's — never the raw page, + # which is where the card/address resources live. + details: dict + + # -- base downloader (shared fetch/validate plumbing) ---------------------- class BaseNativeDownloader: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index c9cd6a1..11bb7da 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -14,6 +14,18 @@ the later step can drive it: - extract_media(post, included_index) → list[MediaItem] - parse_cursor_from_url(url) → cursor +Milestone 387 added a SECOND read path on the same session: the membership +roster — what the ACCOUNT subscribes to, as opposed to what one creator has +posted. + - iter_memberships(user_id) → Iterator[Membership] + - current_user_id() → str + +It is an OPTIONAL seam by construction, probed with +`getattr(client, "iter_memberships", None)` exactly as `post_is_gated` already +is. A client that does not implement it (Discord, HentaiFoundry) makes the +whole feature invisible for that platform — no flag, no config row, no +"unsupported" branch to keep alive. + Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we depend on (top-level `data`, media resources carrying `file_name`/`url`) are the contract. If a response comes back as an HTML login page or a media @@ -41,6 +53,7 @@ from ..utils.paths import filehash_from_url from ..utils.prosemirror import post_body_html from .native_ingest_common import ( _MAX_429_RETRIES, + Membership, NativeAuthError, NativeDriftError, NativeIngestError, @@ -52,8 +65,34 @@ from .native_ingest_common import ( log = logging.getLogger(__name__) _POSTS_URL = "https://www.patreon.com/api/posts" +_MEMBERS_URL = "https://www.patreon.com/api/members" +_CURRENT_USER_URL = "https://www.patreon.com/api/current_user" _TIMEOUT_SECONDS = 30.0 +# --- membership roster contract (#387 C2) --------------------------------- +# Characterized from a real capture of the operator's own session — Scribe note +# #3886. NOT from Patreon's public v2 API, which is the CREATOR api behind +# OAuth scopes and a different surface entirely (project rule 130). +# +# DELIBERATELY MINIMAL, and that is a privacy decision rather than a +# performance one. The web app's own include set pulls `latest_pledge.card` +# and `address`; the card resources come back carrying the ACCOUNT HOLDER'S +# EMAIL in `merchant_name`. Copying the browser's query string wholesale — the +# obvious move — would have FC fetching payment PII it has no use for and can +# only mishandle. We ask for the creator and the tier, and nothing else. +_MEMBERS_INCLUDE = "campaign,reward" +_FIELDS_MEMBER = ( + "patron_status,is_free_member,is_gifted,pledge_amount_cents,currency," + "pledge_cadence,next_charge_date,access_expires_at" +) +_FIELDS_MEMBERS_CAMPAIGN = "name,url,vanity,is_active" +_FIELDS_REWARD = "title" +# The browser sends 1000. Whether a server-side ceiling applies below that is +# untested (note #3886, open question 4), so page conservatively: a wrong guess +# costs one extra request, and the paging loop is driven by meta.pagination +# rather than by this number. +_MEMBERS_PAGE_COUNT = 200 + # JSON:API request contract (observed from real traffic — see module plan). _INCLUDE = ( "campaign,access_rules,attachments,attachments_media,audio,images,media," @@ -182,20 +221,27 @@ class PatreonClient: params["page[cursor]"] = cursor return params - def _fetch(self, campaign_id: str, cursor: str | None) -> dict: + def _request(self, url: str, params: dict[str, str], *, what: str, scope: str) -> dict: + """One paced, retried, error-classified GET returning parsed JSON. + + Extracted from `_fetch` so the membership endpoint (#387 C2) rides the + SAME request path rather than growing a second copy of the 429 backoff, + the auth-vs-drift classification and the Retry-After plumbing. Two + copies of this would drift, and the half that drifted would be the one + that only runs once a day. + + `what` / `scope` only shape the messages ("posts"/"campaign_id=123"), + so a failure still says which call failed and against what. + """ if self._request_sleep > 0: time.sleep(self._request_sleep) # pace the API endpoint attempt = 0 while True: try: - resp = self._session.get( - _POSTS_URL, - params=self._params(campaign_id, cursor), - timeout=_TIMEOUT_SECONDS, - ) + resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS) except requests.RequestException as exc: raise PatreonAPIError( - f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" + f"Patreon {what} request failed ({scope}): {exc}" ) from exc # Transient rate-limit: back off and retry rather than failing the @@ -205,8 +251,8 @@ class PatreonClient: attempt += 1 delay = retry_after_seconds(resp, attempt) log.warning( - "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)", - campaign_id, delay, attempt, self._max_retries, + "Patreon 429 (%s) — backing off %.1fs (retry %d/%d)", + scope, delay, attempt, self._max_retries, ) time.sleep(delay) continue @@ -216,9 +262,8 @@ class PatreonClient: # Auth rejected — expired/missing cookies or an insufficient tier. # Actionable as "rotate credentials", so it's auth, not drift/http. raise PatreonAuthError( - f"Patreon posts API returned HTTP {resp.status_code} — auth " - f"rejected (cookies expired or tier insufficient; " - f"campaign_id={campaign_id})", + f"Patreon {what} API returned HTTP {resp.status_code} — auth " + f"rejected (cookies expired or tier insufficient; {scope})", status_code=resp.status_code, ) if resp.status_code != 200: @@ -234,24 +279,27 @@ class PatreonClient: except (TypeError, ValueError): retry_after = None raise PatreonAPIError( - f"Patreon posts API returned HTTP {resp.status_code} " - f"(campaign_id={campaign_id})", + f"Patreon {what} API returned HTTP {resp.status_code} ({scope})", status_code=resp.status_code, retry_after=retry_after, ) try: - payload = resp.json() + return resp.json() except ValueError as exc: # A non-JSON body here is almost always the HTML login/challenge # page served when cookies are missing/expired — that is an AUTH # failure (rotate cookies), not API drift (update the ingester) and # not a transient network error. raise PatreonAuthError( - "Patreon posts API returned a non-JSON response (likely an " - f"HTML login/challenge page — session expired; " - f"campaign_id={campaign_id}): {exc}" + f"Patreon {what} API returned a non-JSON response (likely an " + f"HTML login/challenge page — session expired; {scope}): {exc}" ) from exc - return payload + + def _fetch(self, campaign_id: str, cursor: str | None) -> dict: + return self._request( + _POSTS_URL, self._params(campaign_id, cursor), + what="posts", scope=f"campaign_id={campaign_id}", + ) # -- parsing ----------------------------------------------------------- @@ -510,6 +558,159 @@ class PatreonClient: return current_cursor = next_cursor + # -- membership roster (#387 C2) --------------------------------------- + + def current_user_id(self) -> str: + """The signed-in account's own numeric user id. + + Needed because `/api/members` is filtered by `filter[user_id]` — the + endpoint answers "who are the members of X", and the account asking + about ITSELF still has to say so. + + INFERRED, NOT CHARACTERIZED. C0 captured `/api/members`, not this; what + is relied on here is only the JSON:API envelope (`data.id`), which this + same API demonstrably uses everywhere else. If that inference is wrong + it raises drift rather than returning something plausible — which is + the right failure, because the alternative is a confidently empty + roster and an empty roster means "cancel everything" to C4. + """ + payload = self._request( + _CURRENT_USER_URL, {"json-api-version": "1.0"}, + what="current_user", scope="self", + ) + data = (payload or {}).get("data") + if not isinstance(data, dict) or not data.get("id"): + raise PatreonDriftError( + "Patreon current_user response had no data.id — cannot scope " + "the membership roster to this account" + ) + return str(data["id"]) + + def _members_params(self, user_id: str | None, offset: int) -> dict[str, str]: + params = { + "include": _MEMBERS_INCLUDE, + "fields[member]": _FIELDS_MEMBER, + "fields[campaign]": _FIELDS_MEMBERS_CAMPAIGN, + "fields[reward]": _FIELDS_REWARD, + "page[offset]": str(offset), + "page[count]": str(_MEMBERS_PAGE_COUNT), + "json-api-version": "1.0", + "json-api-use-default-includes": "false", + } + if user_id: + params["filter[user_id]"] = user_id + # NOTE: `filter[membership_type]` is deliberately NOT sent. The browser + # sends the six values its settings page wants to show, and the capture + # proves that list is NOT the same vocabulary as the `patron_status` + # attribute — a row selected as `free_member` came back with + # `patron_status: former_patron`, a word absent from the filter. Sending + # no filter asks for everything the endpoint will give, which is what a + # roster wants: a membership that DISAPPEARS is the signal C4 reads, and + # a filter tuned for a UI that hides lapses would manufacture exactly + # that disappearance. (Note #3886, open question 1.) + return params + + @staticmethod + def _validate_members_response(response: dict) -> None: + """Drift checks specific to the roster. + + Stricter than the posts path about pagination on purpose: `iter_posts` + can treat a missing `links.next` as "that was the last page", but here + a missing total is indistinguishable from a truncated page — and a + roster that silently stops half way reads downstream as "you cancelled + those", which is the worst wrong answer this feature can give. + """ + PatreonClient._validate_response(response) + meta = response.get("meta") + if not isinstance(meta, dict): + raise PatreonDriftError("Patreon members response missing 'meta'") + pagination = meta.get("pagination") + if not isinstance(pagination, dict) or "total" not in pagination: + raise PatreonDriftError( + "Patreon members response missing meta.pagination.total — " + "cannot tell a complete roster from a truncated one" + ) + + def _membership(self, member: dict, index: dict) -> Membership: + attrs = member.get("attributes") or {} + if "patron_status" not in attrs: + raise PatreonDriftError( + "Patreon member resource has no patron_status attribute" + ) + + campaign_ids = self._related_ids(member, "campaign") + if not campaign_ids: + raise PatreonDriftError( + "Patreon member resource has no campaign relationship — a " + "membership we cannot attribute to a creator is not usable" + ) + campaign_id = campaign_ids[0] + campaign = index.get(("campaign", campaign_id)) or {} + + # A member has at most one reward, and `reward.data` is legitimately + # null — an active patron with no tier. Absence is a fact about the + # membership, not a parse failure. + tier_names: list[str] = [] + for reward_id in self._related_ids(member, "reward"): + title = (index.get(("reward", reward_id)) or {}).get("title") + if title: + tier_names.append(str(title)) + + return Membership( + campaign_id=campaign_id, + display_name=campaign.get("name"), + url=campaign.get("url"), + vanity=campaign.get("vanity"), + status=attrs.get("patron_status"), + # Default False, not None: the attribute is always present in the + # capture, and treating a missing one as "free" would understate + # access rather than overstate it. + is_free_member=bool(attrs.get("is_free_member")), + tier_names=tier_names, + # The MEMBER's amount, never the reward's. `reward.amount_cents` is + # the creator's list price in the CREATOR's currency (the capture + # has CAD, DKK and EUR rewards sitting on USD pledges), so reading + # it would report a number the operator has never been charged. + amount_cents=attrs.get("pledge_amount_cents"), + currency=attrs.get("currency"), + details={"member": attrs, "campaign": campaign}, + ) + + def iter_memberships(self, user_id: str | None = None) -> Iterator[Membership]: + """Yield every membership the account holds. + + Pages on `page[offset]`/`page[count]` against `meta.pagination.total` — + NOT on `links`. The response's own `links.first` is built without the + `/api/` prefix the request uses, so following it verbatim would hit the + web page instead of the API (note #3886). + + `user_id` omitted means the `filter[user_id]` parameter is omitted. + Whether the endpoint then defaults to self is UNTESTED — pass + `current_user_id()` unless you are deliberately probing that. + """ + user_id = user_id or None + offset = 0 + seen = 0 + while True: + response = self._request( + _MEMBERS_URL, self._members_params(user_id, offset), + what="members", scope="membership roster", + ) + self._validate_members_response(response) + index = self._transform(response) + rows = [m for m in (response.get("data") or []) if isinstance(m, dict)] + for member in rows: + yield self._membership(member, index) + + seen += len(rows) + total = int(response["meta"]["pagination"]["total"] or 0) + # An empty page terminates regardless of what `total` claims. Trusting + # the total alone would spin forever against a server that reports + # more rows than it will hand over. + if not rows or seen >= total: + return + offset += len(rows) + # -- detail (full body enrichment) ------------------------------------- def fetch_post_detail_content(self, post_id: str) -> str | None: diff --git a/backend/app/services/platforms/__init__.py b/backend/app/services/platforms/__init__.py index be94f3d..4220dfc 100644 --- a/backend/app/services/platforms/__init__.py +++ b/backend/app/services/platforms/__init__.py @@ -11,7 +11,11 @@ Lifted from GallerySubscriber's and ~/.../extension/lib/platforms.js. Five platforms; auth_type and URL patterns match GS exactly so the existing browser extension hits FC unmodified. deviantart was dropped at #3069 (2026-08-27) — -FC downloaders are art-dedicated services only. +FC downloaders are art-dedicated services only. pixiv was retired at +milestone #406 (2026-09-13, rule #171): unregistered here first, which +switches it off everywhere this registry is consulted; `pixiv.py` and the +pixiv client/downloader/ingester stay in the tree, uncalled, until the +milestone's phase 2 deletes them. """ from .base import ( @@ -22,7 +26,6 @@ from .base import ( from .discord import INFO as _DISCORD from .hentaifoundry import INFO as _HENTAIFOUNDRY from .patreon import INFO as _PATREON -from .pixiv import INFO as _PIXIV from .subscribestar import INFO as _SUBSCRIBESTAR PLATFORMS: dict[str, PlatformInfo] = { @@ -32,7 +35,6 @@ PLATFORMS: dict[str, PlatformInfo] = { _SUBSCRIBESTAR, _HENTAIFOUNDRY, _DISCORD, - _PIXIV, ) } diff --git a/backend/app/services/post_association_service.py b/backend/app/services/post_association_service.py new file mode 100644 index 0000000..8d45f0d --- /dev/null +++ b/backend/app/services/post_association_service.py @@ -0,0 +1,308 @@ +"""The announcement matcher: which Patreon post announced which Discord drop. + +Milestone 388, step E5. + +Two of the operator's artists post a deliberately CROPPED fragment on Patreon +to signal that the real thing has landed in their Discord. This service +proposes those pairs, and proposes only — the operator accepts or dismisses, +following the FC-6.3 series matcher (task 737) rather than linking on its own. + +## Why confirm-only is not caution for its own sake + +A wrongly-asserted association tells the operator that two different pieces are +one. That is strictly worse than no link at all: no link leaves them exactly +where they already were, a wrong one actively misinforms and then propagates +into whatever reads the association. So the matcher's job is to make a SHORT +list worth reading, not a long list worth trusting. + +## Signals, and the one deliberately NOT built + +1. **Time proximity.** The Patreon post exists in order to announce the drop, + so the two are minutes-to-hours apart. Nearly free, and strong. +2. **The post says so.** These announcements routinely name Discord or carry + an invite link, which is close to a declaration. + +3. **Crop-to-source matching is HELD, on the plan's own instruction** — it is + real work with real false-positive risk, and it is only worth building once + 1 and 2 are shown to be insufficient against the operator's actual artists. + Nothing here should be read as evidence it is unnecessary; it is deferred, + and the thing that would justify it is an empty review queue on a pair the + operator can see with their own eyes. + + Note also that a naive whole-image SigLIP similarity is NOT that signal. A + cropped teaser and its full version are exactly the pair a whole-image + comparison handles worst, so adding one as a "bonus" would mostly add noise + while looking like progress. + +## Creator identity comes free, so E4 is not actually a prerequisite + +The plan listed E4 (creator identity across the two channels) as a dependency. +It is not one for the pairs that matter today: a Patreon `Source` and a Discord +`Source` the operator has added under the same `Artist` already share +`Post.artist_id`, and the synthetic grouping inherits it (E2). E4 EXTENDS this +to creators whose association FC has to learn rather than being told; it is not +needed to represent an association FC already knows. + +## A correction the plan carried, worth restating + +`link_extract.py` does exist and does capture off-platform links — but only +file hosts (`SUPPORTED_HOSTS` is mega/gdrive/mediafire/dropbox/pixeldrain). +`host_for()` returns None for a Discord URL, so no `ExternalLink` row is ever +written for one. The declaration signal therefore reads the post body itself +rather than the extracted-links table the plan assumed it could use. +""" + +from __future__ import annotations + +import logging +import re +from datetime import UTC, datetime, timedelta + +from sqlalchemy import func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImportSettings, Post, PostAssociation +from ..utils.text import html_to_plain +from .discord_grouping import DROP_GROUPER + +log = logging.getLogger(__name__) + +# Additive weights, summing to 1.0. Kept as constants rather than settings — +# the sensitivity knob that matters is the threshold, and per-signal weights +# are an over-tune (same call as series_match_service.WEIGHTS). +# +# THE RELATIONSHIP TO THE THRESHOLD IS THE DESIGN. No single weight may reach +# the default threshold, which is what makes "time proximity alone is never +# enough" arithmetic rather than aspirational: on a busy day an artist posts +# several times, and a matcher that could pair on proximity alone would turn +# every busy day into false pairs. A guard test pins this. +WEIGHTS = {"proximity": 0.55, "declared": 0.45} + +# A Discord INVITE in the body is close to a declaration; the bare word is +# weaker but still meaningful, because these posts are short and on-topic. +_INVITE = re.compile(r"discord\.(?:gg|com/invite)/", re.I) +_MENTION = re.compile(r"\bdiscord\b", re.I) + +DECLARED_INVITE = 1.0 +DECLARED_MENTION = 0.6 + +MAX_CANDIDATES = 25 + + +def proximity_signal(gap: timedelta, window: timedelta) -> float: + """1.0 when the two posts are simultaneous, decaying linearly to 0 at the + window's edge. Linear rather than a step, so a pair an hour outside a + hand-tuned window degrades instead of vanishing.""" + if window <= timedelta(0): + return 0.0 + seconds = abs(gap.total_seconds()) + if seconds >= window.total_seconds(): + return 0.0 + return round(1.0 - (seconds / window.total_seconds()), 4) + + +def declared_signal(description: str | None) -> float: + """Does the announcement say, in its own body, that this is about Discord? + + The invite is matched against the RAW body and the bare mention against the + stripped text, which is not fussiness — post bodies are HTML, and these + creators put the invite in an anchor's `href`. `html_to_plain` discards + attributes, so stripping first would have thrown away the strongest form of + the signal and left only whatever the link text happened to say. + + The mention still reads stripped text, so `\bdiscord\b` is matched against + prose rather than against markup and URLs, where it would fire on any link + that merely passes through a discord domain. + """ + if not description: + return 0.0 + if _INVITE.search(description): + return DECLARED_INVITE + if _MENTION.search(html_to_plain(description) or ""): + return DECLARED_MENTION + return 0.0 + + +def weighted_score(signals: dict) -> float: + return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) + + +def _post_time(post: Post) -> datetime: + return post.post_date or post.downloaded_at + + +class PostAssociationService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _decided(self, announcement_id: int) -> set[int]: + """Payload posts already proposed for this announcement, in ANY status. + + Dismissed pairs are included deliberately: re-proposing a pair the + operator has already rejected on every subsequent scan is the single + behaviour that makes a review queue get ignored. + """ + rows = (await self.session.execute( + select(PostAssociation.payload_post_id) + .where(PostAssociation.announcement_post_id == announcement_id) + )).scalars().all() + return set(rows) + + async def _candidate_groups( + self, announcement: Post, *, window: timedelta, + ) -> list[Post]: + """Synthetic Discord groupings by the SAME artist, inside the window. + + Same-artist is the identity signal and it is free (see the module + docstring on E4). It is also a hard filter rather than a scored one: + two different creators posting minutes apart is a coincidence, not + evidence, and letting it score at all would mean a busy hour across the + library could out-vote everything else. + """ + at = _post_time(announcement) + sort_key = func.coalesce(Post.post_date, Post.downloaded_at) + return (await self.session.execute( + select(Post) + .where( + Post.artist_id == announcement.artist_id, + Post.synthesized_by == DROP_GROUPER, + Post.id != announcement.id, + sort_key >= at - window, + sort_key <= at + window, + ) + .order_by(sort_key) + .limit(MAX_CANDIDATES) + )).scalars().all() + + async def match_post( + self, announcement_id: int, *, threshold: float, window_hours: float, + ) -> int: + """Score one announcement against nearby groupings. Returns proposals made.""" + announcement = await self.session.get(Post, announcement_id) + if announcement is None or announcement.synthesized_by is not None: + # A synthetic post cannot announce anything — FC wrote it. + return 0 + + window = timedelta(hours=window_hours) + declared = declared_signal(announcement.description) + already = await self._decided(announcement_id) + + made = 0 + for group in await self._candidate_groups(announcement, window=window): + if group.id in already: + continue + signals = { + "proximity": proximity_signal( + _post_time(group) - _post_time(announcement), window, + ), + "declared": declared, + } + score = weighted_score(signals) + if score < threshold: + continue + self.session.add(PostAssociation( + announcement_post_id=announcement.id, + payload_post_id=group.id, + score=score, + signals=signals, + status="pending", + )) + made += 1 + return made + + async def list_pending(self) -> list[dict]: + rows = (await self.session.execute( + select(PostAssociation) + .where(PostAssociation.status == "pending") + .order_by(PostAssociation.score.desc(), PostAssociation.id.desc()) + )).scalars().all() + return [ + { + "id": a.id, + "announcement_post_id": a.announcement_post_id, + "payload_post_id": a.payload_post_id, + "score": a.score, + "signals": a.signals, + } + for a in rows + ] + + async def accept(self, association_id: int) -> dict | None: + a = await self.session.get(PostAssociation, association_id) + if a is None: + return None + a.status = "linked" + return {"id": a.id, "status": a.status} + + async def dismiss(self, association_id: int) -> dict | None: + a = await self.session.get(PostAssociation, association_id) + if a is None: + return None + # Kept, not deleted — the row is what remembers the rejection. + a.status = "dismissed" + return {"id": a.id, "status": a.status} + + async def linked_for(self, post_ids: list[int]) -> dict[int, list[dict]]: + """Accepted links touching these posts, keyed by post id, BOTH ways. + + A post is either end of the relationship, and each end wants the other + one: the teaser wants "the full set is over here", the grouping wants + "this is what announced me". One query, both directions. + """ + if not post_ids: + return {} + rows = (await self.session.execute( + select(PostAssociation).where( + PostAssociation.status == "linked", + or_( + PostAssociation.announcement_post_id.in_(post_ids), + PostAssociation.payload_post_id.in_(post_ids), + ), + ) + )).scalars().all() + out: dict[int, list[dict]] = {} + for a in rows: + if a.announcement_post_id in post_ids: + out.setdefault(a.announcement_post_id, []).append( + {"role": "announces", "post_id": a.payload_post_id, "id": a.id} + ) + if a.payload_post_id in post_ids: + out.setdefault(a.payload_post_id, []).append( + {"role": "announced_by", "post_id": a.announcement_post_id, "id": a.id} + ) + return out + + +async def rescan(session: AsyncSession, *, now: datetime | None = None) -> dict: + """Score every recent non-synthetic post against nearby groupings.""" + settings = await ImportSettings.load(session) + if not settings.discord_link_enabled: + return {"enabled": False, "scanned": 0, "proposed": 0} + + now = now or datetime.now(UTC) + window_hours = float(settings.discord_link_window_hours) + # Only look at announcements that could still have a partner in range — + # a full-library rescan is the manual button's job, not the sweep's. + horizon = now - timedelta(hours=window_hours * 2) + sort_key = func.coalesce(Post.post_date, Post.downloaded_at) + ids = (await session.execute( + select(Post.id).where( + Post.synthesized_by.is_(None), + Post.absorbed_by_post_id.is_(None), + sort_key >= horizon, + ) + )).scalars().all() + + svc = PostAssociationService(session) + proposed = 0 + for pid in ids: + proposed += await svc.match_post( + pid, + threshold=float(settings.discord_link_threshold), + window_hours=window_hours, + ) + log.info( + "discord announcement matcher: scanned %d post(s), proposed %d pair(s)", + len(ids), proposed, + ) + return {"enabled": True, "scanned": len(ids), "proposed": proposed} diff --git a/backend/app/services/post_body.py b/backend/app/services/post_body.py new file mode 100644 index 0000000..127e4f0 --- /dev/null +++ b/backend/app/services/post_body.py @@ -0,0 +1,93 @@ +"""Rendering a post's stored HTML body for display — sanitized, and pointing +at our own copies of the images rather than the platform's. + +## Why this is one function and not two + +Two surfaces render a post body: the post detail view +(`PostFeedService.get_post`) and the provenance panel (`ProvenanceService`). +Until issue #3965 the detail view did `_localize_inline_images(sanitize(...))` +and provenance did `sanitize(...)` alone — so the provenance panel hotlinked +the platform CDN for images FC had already downloaded, which is exactly what +#830 Phase 2 set out to stop. + +That bug was available because the two halves were separately callable and +only one of them looked mandatory. `render_post_body` is therefore the whole +pipeline in a single call, and it is the only thing callers are meant to +reach for: sanitizing without localizing is not a supported operation, so it +should not be a reachable one. `localize_inline_images` stays public only +because a caller that already holds sanitized HTML needs it. + +## Cost + +`localize_inline_images` issues ZERO queries for a body with no inline +``, which is most of them — it returns before touching the session if +the body is empty, has no image tags, or has none carrying a parseable CDN +filehash. That early exit is why callers may run this per post in a loop +rather than needing a batched form; do not "optimize" it into one without a +measurement saying the loop is actually hot. +""" + +from __future__ import annotations + +from html import unescape + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord +from ..utils.html_sanitize import extract_img_srcs, rewrite_img_srcs, sanitize_post_html +from ..utils.paths import filehash_from_url +from .gallery_service import image_url + + +async def render_post_body( + session: AsyncSession, description: str | None, artist_id: int | None, +) -> str | None: + """A post's body, ready to put in front of someone. + + Sanitize, then repoint inline images at local copies. Use this rather than + calling either half on its own — see the module docstring. + """ + return await localize_inline_images( + session, sanitize_post_html(description), artist_id, + ) + + +async def localize_inline_images( + session: AsyncSession, html: str | None, artist_id: int | None, +) -> str | None: + """Rewrite a post body's inline `` to locally-served copies. + + The join key is the CDN filehash the downloader persisted on each + ImageRecord (source_filehash): for every body image whose filehash maps + to a stored image of THIS artist, swap the src to /images/. Images + we never captured (or pre-Phase-2 rows with no filehash) are left as-is — + they keep hotlinking, which is the prior behavior. Scoped to the post's + artist so one creator's body never resolves to another's file. + """ + if not html or artist_id is None: + return html + srcs = extract_img_srcs(html) + if not srcs: + return html + # filehash -> the raw (as-in-HTML) src strings carrying it. A body can + # repeat the same image; keep every raw form so each is substituted. + by_hash: dict[str, list[str]] = {} + for raw in srcs: + fh = filehash_from_url(unescape(raw)) + if fh: + by_hash.setdefault(fh, []).append(raw) + if not by_hash: + return html + rows = (await session.execute( + select(ImageRecord.source_filehash, ImageRecord.path) + .where( + ImageRecord.artist_id == artist_id, + ImageRecord.source_filehash.in_(list(by_hash)), + ) + )).all() + replace: dict[str, str] = {} + for fh, path in rows: + for raw in by_hash.get(fh, ()): + replace[raw] = image_url(path) + return rewrite_img_srcs(html, replace) diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index d4aec83..6207667 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -11,8 +11,6 @@ attachments from PostAttachment) so the API layer can jsonify directly. """ from __future__ import annotations -from html import unescape - from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -26,23 +24,50 @@ from ..models import ( Source, attachment_download_url, ) -from ..utils.html_sanitize import ( - extract_img_srcs, - rewrite_img_srcs, - sanitize_post_html, -) -from ..utils.paths import filehash_from_url from ..utils.text import html_to_plain, truncate_at_word -from .gallery_service import image_url, thumbnail_url +from .gallery_service import thumbnail_url from .pagination import decode_cursor, encode_cursor +from .post_body import render_post_body DESCRIPTION_LIMIT = 280 THUMBNAIL_LIMIT = 6 def _sort_key(): - """Postgres COALESCE expression used in ORDER BY and WHERE clauses.""" - return func.coalesce(Post.post_date, Post.downloaded_at) + """Postgres COALESCE expression used in ORDER BY and WHERE clauses. + + `resurfaced_at` leads (milestone 388 E3). A synthetic post stays OPEN — a + creator who adds variants the next day extends the existing post — so such + a post has two dates, and which one orders the feed is a real decision: + + * ordering by when the drop STARTED buries a group that grows a week later + under a week of other posts, so the operator never sees the new content — + which defeats keeping the group open at all; + * ordering by every growth lets a group that gains one image a day sit + permanently at the top, so chat out-competes authored posts for the front + page — the opposite of "post pacing stays front and centre". + + So the feed orders by neither directly. `resurfaced_at` moves only when the + anti-thrash rule fires (discord_grouping.should_resurface: enough new + images AND enough time since the last move), which means a drip-feed + updates IN PLACE and a genuine second wave resurfaces exactly once. + + It is NULL on every ordinary post, so this COALESCE cannot move anything + that is not a grouping. Used identically in ORDER BY and in the cursor's + WHERE, which is what keeps pagination stable across the change. + """ + return func.coalesce(Post.resurfaced_at, Post.post_date, Post.downloaded_at) + + +def _post_sort_value(post: Post): + """The Python twin of `_sort_key()`, for building a cursor from a loaded row. + + Kept next to it on purpose: these two are one expression in two languages, + and the failure when they disagree is not an error but a quiet one — rows + skipped or repeated at page boundaries, which reads as a backend bug + anywhere but here. + """ + return post.resurfaced_at or post.post_date or post.downloaded_at class PostFeedService: @@ -86,6 +111,13 @@ class PostFeedService: .join(Artist, Post.artist_id == Artist.id) .outerjoin(Source, Post.source_id == Source.id) ) + # Absorbed posts are the individual chat messages a synthetic post + # replaced (milestone 388 E2). They stay in the table — they are the + # images' true origin and the grouping has to be auditable — but the + # feed shows the post FC authored, not the dozen lines it was built + # from. `around` and `get_post` deliberately do NOT apply this: reaching + # a member by id is how you inspect a grouping. + stmt = stmt.where(Post.absorbed_by_post_id.is_(None)) if artist_id is not None: stmt = stmt.where(Post.artist_id == artist_id) if platform is not None: @@ -127,15 +159,19 @@ class PostFeedService: # Far edge in the travel direction: oldest row going older, # newest row going newer (rows is descending for display). edge_post = rows[-1][0] if direction == "older" else rows[0][0] - edge_key = edge_post.post_date or edge_post.downloaded_at + # Must match _sort_key() exactly, including resurfaced_at's + # precedence: a cursor built from a different expression than the + # ORDER BY silently skips or repeats rows at every page boundary. + edge_key = _post_sort_value(edge_post) next_cursor = encode_cursor(edge_key, edge_post.id) post_ids = [p.id for p, _, _ in rows] thumbs_map = await self._thumbnails_for(post_ids) atts_map = await self._attachments_for(post_ids) + links_map = await self._links_for(post_ids) items = [ - self._to_dict(post, artist, source, thumbs_map, atts_map) + self._to_dict(post, artist, source, thumbs_map, atts_map, links_map) for post, artist, source in rows ] return {"items": items, "next_cursor": next_cursor} @@ -161,7 +197,7 @@ class PostFeedService: if anchor is None: return None anchor_post, anchor_artist, anchor_source = anchor - anchor_key = anchor_post.post_date or anchor_post.downloaded_at + anchor_key = _post_sort_value(anchor_post) anchor_cursor = encode_cursor(anchor_key, anchor_post.id) older = await self.scroll( @@ -176,6 +212,7 @@ class PostFeedService: atts_map = await self._attachments_for([anchor_post.id]) anchor_item = self._to_dict( anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map, + await self._links_for([anchor_post.id]), ) return { "items": newer["items"] + [anchor_item] + older["items"], @@ -199,58 +236,25 @@ class PostFeedService: # the default arg. thumbs_map = await self._thumbnails_for([post.id], limit=None) atts_map = await self._attachments_for([post.id]) - item = self._to_dict(post, artist, source, thumbs_map, atts_map) + item = self._to_dict( + post, artist, source, thumbs_map, atts_map, + await self._links_for([post.id]), + ) item["description_full"] = html_to_plain(post.description) # Full (uncapped) translated description for the detail view (#143). item["description_translated_full"] = post.description_translated - # Sanitized HTML body for faithful (semantic) rendering in the post view; - # detail-only (the feed list stays lightweight plain text). None when the - # post has no body. Inline `` sources are remapped to locally-served - # copies (#830 Phase 2) so the body never hotlinks the public CDN. - item["description_html"] = await self._localize_inline_images( - sanitize_post_html(post.description), post.artist_id, + # Rendered body for faithful (semantic) display in the post view; + # detail-only, which is what keeps the feed list lightweight plain text + # (measured: note #3962). None when the post has no body. What + # "rendered" involves — sanitize, then repoint inline images at our own + # copies — belongs to `render_post_body`, which the provenance panel + # shares so the two surfaces cannot drift apart again (#3965). + item["description_html"] = await render_post_body( + self.session, post.description, post.artist_id, ) item["external_links"] = await self._external_links_for(post.id) return item - async def _localize_inline_images( - self, html: str | None, artist_id: int | None, - ) -> str | None: - """Rewrite a post body's inline `` to locally-served copies. - - The join key is the CDN filehash the downloader persisted on each - ImageRecord (source_filehash): for every body image whose filehash maps - to a stored image of THIS artist, swap the src to /images/. Images - we never captured (or pre-Phase-2 rows with no filehash) are left as-is — - they keep hotlinking, which is the prior behavior. Scoped to the post's - artist so one creator's body never resolves to another's file.""" - if not html or artist_id is None: - return html - srcs = extract_img_srcs(html) - if not srcs: - return html - # filehash -> the raw (as-in-HTML) src strings carrying it. A body can - # repeat the same image; keep every raw form so each is substituted. - by_hash: dict[str, list[str]] = {} - for raw in srcs: - fh = filehash_from_url(unescape(raw)) - if fh: - by_hash.setdefault(fh, []).append(raw) - if not by_hash: - return html - rows = (await self.session.execute( - select(ImageRecord.source_filehash, ImageRecord.path) - .where( - ImageRecord.artist_id == artist_id, - ImageRecord.source_filehash.in_(list(by_hash)), - ) - )).all() - replace: dict[str, str] = {} - for fh, path in rows: - for raw in by_hash.get(fh, ()): - replace[raw] = image_url(path) - return rewrite_img_srcs(html, replace) - async def _external_links_for(self, post_id: int) -> list[dict]: """Off-platform file-host links recorded for a post (detail-only). Each carries its host, full url, label, and download status so the post view @@ -365,9 +369,25 @@ class PostFeedService: }) return out + async def _links_for(self, post_ids: list[int]) -> dict[int, list[dict]]: + """Accepted announcement links touching these posts (#388 E5). + + Only ACCEPTED ones. A pending proposal is a question for the review + queue, not a claim to render beside the artwork — showing one here + would assert a link the operator has not agreed to, which is the exact + failure the confirm-only design exists to prevent. + """ + # Imported here rather than at module scope: post_association_service + # imports discord_grouping, which imports the models, and the feed + # service is imported by the API at startup. A local import keeps that + # chain out of the import graph for a purely optional read. + from .post_association_service import PostAssociationService + + return await PostAssociationService(self.session).linked_for(post_ids) + def _to_dict( self, post: Post, artist: Artist, source: Source | None, - thumbs_map: dict, atts_map: dict, + thumbs_map: dict, atts_map: dict, links_map: dict | None = None, ) -> dict: plain_full = html_to_plain(post.description) if post.description else None if plain_full is None: @@ -400,6 +420,27 @@ class PostFeedService: "translated_source_lang": post.translated_source_lang, # Sticky per-post translation choice (auto/force/original, #155). "translation_override": post.translation_override, + # Milestone 388 E2. Non-null means FC AUTHORED this post by grouping + # a creator's drop — the UI must say so wherever the post appears, + # and `synthesis` carries what it was built from so the operator can + # audit a grouping FC invented. Null for every real post; the two + # keys are always present so the frontend never branches on absence. + "synthesized_by": post.synthesized_by, + "synthesis": post.synthesis_details, + # #388 E3. A grouping stays open, so the card can say "updated N + # ago" — which is the whole signal that chat content is trickling + # in. NULL means it has not grown since it was created. + "last_grew_at": post.last_grew_at.isoformat() if post.last_grew_at else None, + # Accepted links only (#388 E5): [{role, post_id, id}], where role + # is "announces" (this post is the teaser) or "announced_by" (this + # post is the drop). Always a list so the UI never branches on + # absence. + "associations": (links_map or {}).get(post.id, []), + # Non-null on a chat message a synthetic post absorbed. The feed + # filters these out, but `around`/`get_post` still reach them, and + # the UI uses this to explain why a post it linked to is not in the + # stream. + "absorbed_by_post_id": post.absorbed_by_post_id, "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, "source": ( {"id": source.id, "platform": source.platform} diff --git a/backend/app/services/provenance_service.py b/backend/app/services/provenance_service.py index 02f1a64..7b70688 100644 --- a/backend/app/services/provenance_service.py +++ b/backend/app/services/provenance_service.py @@ -18,17 +18,32 @@ from ..models import ( Source, attachment_download_url, ) -from ..utils.html_sanitize import sanitize_post_html +from .post_body import render_post_body -def _post_dict(p: Post) -> dict: +async def _post_dict(session: AsyncSession, p: Post) -> dict: + """One provenance entry's post. + + NOTE: the key names here deliberately differ from + `PostFeedService._to_dict` (`url`/`title`/`date` vs + `post_url`/`post_title`/`post_date`, `attachment_count` vs `attachments`), + and `description_translated` is the FULL text here where the feed truncates + it to DESCRIPTION_LIMIT. That divergence is issue #3965's wider half and is + deliberately NOT addressed here — renaming is a breaking payload change for + ProvenancePanel with no second reason to spend it. + + What IS fixed here is the body: this used to call `sanitize_post_html` + alone, so provenance bodies hotlinked the platform CDN for images already + on disk while the post detail view served local copies. `render_post_body` + is the whole pipeline, so the two surfaces cannot drift apart again. + """ return { "id": p.id, "external_post_id": p.external_post_id, "url": p.post_url, "title": p.post_title, "date": p.post_date.isoformat() if p.post_date else None, - "description_html": sanitize_post_html(p.description), + "description_html": await render_post_body(session, p.description, p.artist_id), "attachment_count": p.attachment_count, # Translation (#143): the English title/description shown by default when # a translation exists; the UI toggles to the original. Source lang labels @@ -131,7 +146,7 @@ class ProvenanceService: "provenance_id": ip.id, "captured_at": ip.captured_at.isoformat() if ip.captured_at else None, - "post": _post_dict(post), + "post": await _post_dict(self.session, post), "source": _source_dict(src) if src is not None else None, "artist": _artist_dict(art), } @@ -154,7 +169,7 @@ class ProvenanceService: return None post, src, art = row return { - "post": _post_dict(post), + "post": await _post_dict(self.session, post), "source": _source_dict(src) if src is not None else None, "artist": _artist_dict(art), "attachments": await self._attachments_for_posts([post.id]), diff --git a/backend/app/services/scheduler_service.py b/backend/app/services/scheduler_service.py index adda050..6d6de11 100644 --- a/backend/app/services/scheduler_service.py +++ b/backend/app/services/scheduler_service.py @@ -8,12 +8,13 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from ..models import AppSetting, Artist, ImportSettings, Source +from .db_helpers import failing_sources_clause, no_access_sources_clause MIN_INTERVAL_SECONDS = 60 MAX_INTERVAL_SECONDS = 86400 @@ -219,10 +220,38 @@ async def scheduler_status(session: AsyncSession) -> dict: cooldowns = await active_platform_cooldowns(session) + # Ingestion health for the front-door ribbon (#387 B3). Counted over ENABLED + # sources rather than the auto_check subset walked above: a source that is + # erroring or paywalled is worth surfacing whether or not a schedule happens + # to poll it. Two scalar COUNTs, not a second pass over `rows`. + # + # Both predicates are the shared ones, so the ribbon and the surfaces it + # links to cannot disagree about what they are counting. + failing_sources = (await session.execute( + select(func.count()).select_from(Source) + .where(Source.enabled.is_(True), failing_sources_clause()) + )).scalar_one() + no_access_sources = (await session.execute( + select(func.count()).select_from(Source) + .where(Source.enabled.is_(True), no_access_sources_clause()) + )).scalar_one() + # #387 B4: lets the front door tell "nothing configured yet" (a fresh + # install — show the on-ramp) apart from "configured, still fetching" (a + # first run in progress — show what's running). Telling someone to add a + # source when they already have three and are mid-backfill is worse than + # saying nothing. Deliberately NOT auto_sources, which counts only what is + # on a schedule: a source with auto_check off still means "configured". + total_sources = (await session.execute( + select(func.count()).select_from(Source).where(Source.enabled.is_(True)) + )).scalar_one() + return { "last_tick_at": last_tick_at, "next_due_at": next_due_at.isoformat() if next_due_at else None, "due_now": due_now, "auto_sources": len(rows), + "failing_sources": failing_sources, + "no_access_sources": no_access_sources, + "total_sources": total_sources, "platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()}, } diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index dcc66d5..22a49ba 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -10,12 +10,16 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..models import ( Artist, + DownloadEvent, ImageProvenance, ImageRecord, ImportSettings, Post, Source, ) +from .db_helpers import failing_sources_clause +from .gallery_dl import ErrorType +from .membership_roster import gated_reasons_for_sources from .platforms import known_platform_keys from .scheduler_service import compute_next_check_at @@ -84,6 +88,17 @@ class SourceRecord: # plan #704: cumulative posts processed across the walk's chunks — live # progress for the badge. backfill_posts: int + # Milestone #387 A3: posts the last walk skipped because the account can't + # view them. Lives on the EVENT (run_stats.tier_gated_count), not the + # source, so it is joined in by `list()` only — None everywhere else, which + # the UI renders as the bare no-access state with no fabricated number. + tier_gated_count: int | None = None + # Milestone #387 C5: WHY those posts are out of reach, when the learned + # roster can say — "lapsed" / "tier" / "free", or None for "no words beyond + # the count". Joined in by `list()` beside the count, and for the same + # reason: it annotates a gated state rather than producing one. Nothing on + # a fetch path may read it (see `membership_roster.gated_reason`). + gated_reason: str | None = None def to_dict(self) -> dict: return { @@ -107,6 +122,8 @@ class SourceRecord: "backfill_bypass_seen": self.backfill_bypass_seen, "backfill_recapture": self.backfill_recapture, "backfill_posts": self.backfill_posts, + "tier_gated_count": self.tier_gated_count, + "gated_reason": self.gated_reason, } @@ -114,6 +131,24 @@ class SourceRecord: _EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"} +# `config_overrides` carries two unrelated things under one column: the +# operator's per-source download settings, and state FC writes for ITSELF. An +# operator edit replaces the first wholesale — removing a key has to be able to +# remove it — but it must never take the second with it. +# +# Two families, both matching data already on disk: +# `_*` the #693 backfill state machine (_backfill_state, _cursor, +# _cursor_stalls, _chunks, _posts, _bypass_seen, _recapture) +# `*_campaign_id` the resolved platform identity cache, written by +# `download_service._phase3_persist` +_CAMPAIGN_ID_SUFFIX = "_campaign_id" + + +def _is_app_managed(key: str) -> bool: + """Is this a key FC maintains, rather than one the operator edits?""" + return key.startswith("_") or key.endswith(_CAMPAIGN_ID_SUFFIX) + + # Plan #693: backfill safety cap. "Start backfill" (and a newly created # enabled source) arms a run-until-done walk; this caps how many time-boxed # chunks it may spend before pausing as "stalled", so a pathological walk that @@ -156,11 +191,67 @@ class SourceService: raise InvalidConfigError("config_overrides must be a JSON object") return config + @staticmethod + def _merged_config(source: Source, incoming: dict | None) -> dict | None: + """The operator's keys replace wholesale; FC's own keys survive. + + Without this, one edit in the Subscriptions dialog silently discarded + the resolved campaign id AND the entire backfill position — the dialog + posts the whole object back (`SourceFormDialog`), so anything absent + from its JSON box was simply gone. + + FC's keys are applied LAST so they win: the dialog round-trips whatever + it last read, and a client echoing a stale `_backfill_state` must not be + able to overwrite what the walk has since written. + """ + managed = { + k: v for k, v in (source.config_overrides or {}).items() + if _is_app_managed(k) + } + if incoming is None: + # An explicit null clears the operator's settings. It is not a + # request to forget where a backfill had got to. + return managed or None + operator = {k: v for k, v in incoming.items() if not _is_app_managed(k)} + return {**operator, **managed} + async def _load_settings(self) -> ImportSettings: return await ImportSettings.load(self.session) + async def _tier_gated_counts(self, source_ids: list[int]) -> dict[int, int]: + """Latest walk's tier-gated post count, per source, in ONE query. + + Selects the `run_stats` sub-object rather than whole `metadata` blobs: + those carry truncated stdout/stderr up to 500KB each, and pulling one + per source to read a single integer would make the subscriptions list + pay for the Logs view. DISTINCT ON + ORDER BY takes the newest event per + source (Postgres-only, like the rest of this codebase). + + Callers pass only the sources that actually need it — the count is + meaningless for a source that isn't tier-gated. + """ + if not source_ids: + return {} + rows = (await self.session.execute( + select( + DownloadEvent.source_id, + DownloadEvent.metadata_["run_stats"], + ) + .where(DownloadEvent.source_id.in_(source_ids)) + .distinct(DownloadEvent.source_id) + .order_by(DownloadEvent.source_id, DownloadEvent.started_at.desc()) + )).all() + counts: dict[int, int] = {} + for source_id, run_stats in rows: + n = (run_stats or {}).get("tier_gated_count") or 0 + if n: + counts[source_id] = int(n) + return counts + def _build_record( self, source: Source, artist: Artist, settings: ImportSettings, + gated_counts: dict[int, int] | None = None, + gated_reasons: dict[int, str] | None = None, ) -> SourceRecord: nxt = compute_next_check_at(source, artist, settings) co = source.config_overrides or {} @@ -185,6 +276,8 @@ class SourceService: backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), backfill_recapture=bool(co.get("_backfill_recapture")), backfill_posts=int(co.get("_backfill_posts", 0)), + tier_gated_count=(gated_counts or {}).get(source.id), + gated_reason=(gated_reasons or {}).get(source.id), ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -210,14 +303,24 @@ class SourceService: stmt = stmt.where(~Source.url.like("sidecar:%")) if failing: # Worst-first so the rollup card surfaces the loudest failures. - stmt = stmt.where(Source.consecutive_failures > 0).order_by( + # Shared clause: the front-door ribbon counts with the same one, so + # it can never report a number this list then contradicts. + stmt = stmt.where(failing_sources_clause()).order_by( Source.consecutive_failures.desc(), Artist.name.asc(), ) else: stmt = stmt.order_by(Artist.name.asc(), Source.id.asc()) rows = (await self.session.execute(stmt)).all() settings = await self._load_settings() - return [self._build_record(s, a, settings) for s, a in rows] + # Only tier-gated rows need either join — on a healthy library that is + # an empty list and both helpers short-circuit without a query. + gated = [s for s, _a in rows if s.error_type == ErrorType.TIER_LIMITED] + gated_counts = await self._tier_gated_counts([s.id for s in gated]) + gated_reasons = await gated_reasons_for_sources(self.session, gated) + return [ + self._build_record(s, a, settings, gated_counts, gated_reasons) + for s, a in rows + ] async def get(self, source_id: int) -> SourceRecord | None: source = (await self.session.execute( @@ -301,10 +404,38 @@ class SourceService: if "url" in fields: fields["url"] = self._validate_url(fields["url"]) if "config_overrides" in fields: - fields["config_overrides"] = self._validate_config(fields["config_overrides"]) + fields["config_overrides"] = self._merged_config( + source, self._validate_config(fields["config_overrides"]) + ) + + # Computed BEFORE the setattr loop, while `source.url` is still the old + # one. See the invalidation below. + url_changed = "url" in fields and fields["url"] != source.url for key, value in fields.items(): setattr(source, key, value) + + if url_changed: + # Repointing a source at a different creator makes a cached campaign + # id WRONG, not merely stale, and `patreon_resolver` consults that + # cache BEFORE attempting any lookup — so a kept id would resolve the + # old creator forever, and the membership join (#387 C4) would report + # a confident wrong match. + # + # This needs saying explicitly only because of the merge above: until + # then the wholesale overwrite wiped the id as an accident of the + # bug, which masked this. Preserving the id makes the invalidation + # this service's job. + # + # The backfill cursor is deliberately NOT cleared. It is opaque + # platform state the walk already validates with its own stall + # guard, and dropping it would restart a long backfill over a + # cosmetic URL edit (http->https, adding `/c/`). + co = dict(source.config_overrides or {}) + for key in [k for k in co if k.endswith(_CAMPAIGN_ID_SUFFIX)]: + co.pop(key) + source.config_overrides = co + # Disabling a source clears its failure state (operator: disable the subs # you're not paying for without them lingering as "failing"). Re-enabling # then starts clean; the next real run re-derives health. Only on the diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index ffdedba..e51e026 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -43,6 +43,7 @@ import requests from ..utils.paths import filehash_from_url from .native_ingest_common import ( _MAX_429_RETRIES, + Membership, NativeAuthError, NativeDriftError, NativeIngestError, @@ -297,6 +298,206 @@ def _extract_creator_name(html: str) -> str | None: return name or None +# -- membership roster (#387 D1) ------------------------------------------ +# +# Characterized from a live operator capture of the account's /subscriptions +# page, 2026-09-13 — Scribe note #3989. Read that note before changing any of +# this; each constant below is a finding from it, not a guess. + +# The account page is fetched from `.adult`. The `.art` age wall never clears +# with the 18+ cookie for FC's requests (see _normalize_ss_host, issues #1259 / +# #1284). The capture itself came from `.art` only because a human had clicked +# through the gate in the browser. +_ROSTER_BASE = "https://subscribestar.adult" +_ROSTER_URL = f"{_ROSTER_BASE}/subscriptions" + +# Two tables, and WHICH table a creator sits in is the only status the page +# gives — there is no per-row status word. Keyed on each card's +# `data-identifier`, the one vocabulary that names a state: the table class +# inside the cancelled card says `for-unsubscribed_users`, a different word for +# the same list (note #3989, CORRECTION 1). The identifier is stored verbatim as +# Membership.status and mapped in membership_roster.MEMBERSHIP_STATUS. +_ROSTER_ACTIVE = "active_subscriptions" +_ROSTER_CANCELLED = "cancelled_subscriptions" + +_ROSTER_ROW_OPEN = '' +# Active rows nest a second `` INSIDE the row's own +# — a narrow-screen duplicate of the actions cell. Its s are not +# columns, so every row is cut here before its cells are read. +_ROSTER_NESTED_ROW = ']*>([^<]*)") +_ROSTER_HEAD_RE = re.compile(r']*>(.*?)', re.DOTALL) +_ROSTER_CELL_RE = re.compile(r']*>(.*?)', re.DOTALL) +_ROSTER_PAGE_LINK_RE = re.compile(r'href="[^"]*[?&]page=\d') +_TAG_RE = re.compile(r"<[^>]+>") + +# Columns that hold identity or controls rather than facts about the +# subscription, so they stay out of `details`. Matched on the header's own text, +# lowercased — the page's words, not ours. +_ROSTER_SKIP_COLUMNS = frozenset({"profile", "updates", "actions"}) + + +def _cell_text(fragment: str) -> str: + """Visible text of a cell: tags dropped, entities decoded, whitespace folded. + + Decoding matters here specifically: an active row with no Discord link + renders its cell as the entity `—`, not as an empty cell. + """ + return " ".join(unescape(_TAG_RE.sub(" ", fragment)).split()) + + +def _roster_table(html: str, identifier: str) -> tuple[str, str] | None: + """One roster card: (its table markup, whatever trails `` inside it). + + None when the card is absent. The trailing part is returned rather than + discarded because it is the pagination check: in the characterized page a + card closes the moment its table does. + """ + start = html.find(f'data-identifier="{identifier}"') + if start < 0: + return None + end = html.find("", start) + if end < 0: + raise SubscribeStarDriftError( + f"SubscribeStar roster card {identifier!r} has no table" + ) + close = html.find("", end) + trailing = html[end + len(""): close if close >= 0 else len(html)] + return html[start:end], trailing + + +def _roster_rows(table: str, identifier: str, base: str) -> list[Membership]: + labels = [_cell_text(h).lower() for h in _ROSTER_HEAD_RE.findall(table)] + body = table[table.find(""):] if "" in table else "" + starts = [m.start() for m in re.finditer(re.escape(_ROSTER_ROW_OPEN), body)] + rows = [] + for n, start in enumerate(starts): + row = body[start: starts[n + 1] if n + 1 < len(starts) else len(body)] + row = row.split(_ROSTER_NESTED_ROW, 1)[0] + + href = _ROSTER_HREF_RE.search(row) + if href is None: + raise SubscribeStarDriftError( + f"SubscribeStar roster row in {identifier!r} has no creator link" + ) + # The creator's numeric id, NOT the slug, is the key (note #3989, + # CORRECTION 2). A slug re-keys when a creator renames; the old row then + # stops appearing, and a disappearance is exactly what reconciliation + # reads as a lapse. The id survives a rename. + user_id = _ROSTER_USER_ID_RE.search(row) + if user_id is None or not user_id.group(1).isdigit(): + raise SubscribeStarDriftError( + f"SubscribeStar roster row in {identifier!r} has no numeric " + f"data-user-id — a membership that cannot be attributed to a " + f"creator is not usable" + ) + name = _ROSTER_NAME_RE.search(row) + slug = unescape(href.group(1)) + cells = _ROSTER_CELL_RE.findall(row) + if len(cells) != len(labels): + # Canary, not a refusal. Identity above does not depend on columns, + # so a shifted column must not fail the whole roster — but it would + # silently mislabel `details` (a price filed under "discord"), so + # say so in the worker log where it is diagnosable. + log.warning( + "SubscribeStar roster %r: %d cells against %d headers — column " + "details may be mislabelled; markup likely changed (note #3989)", + identifier, len(cells), len(labels), + ) + rows.append(Membership( + campaign_id=user_id.group(1), + display_name=(_cell_text(name.group(1)) if name else "") or None, + url=f"{base}/{slug}", + vanity=slug, + status=identifier, + # No free-follow concept on this page (#3970 §2: False when a + # platform has none). + is_free_member=False, + # Tier names live behind a per-row modal, not inline. Fetching every + # modal would be N authenticated requests for a field nothing reads. + tier_names=[], + # Deliberately NOT parsed from the price cell: a bare `$` names no + # currency, and a page price is not proven to be the charge (#3970 + # finding 4). None keeps "unknown" distinct from zero. The raw text + # is kept in `details`. + amount_cents=None, + currency=None, + details={ + # Paired with the header text by POSITION: two columns share the + # `for-date` class, and the updates column's does not carry + # its 's class at all. + "columns": { + label: _cell_text(cell) + for label, cell in zip(labels, cells, strict=False) + if label not in _ROSTER_SKIP_COLUMNS + }, + }, + )) + return rows + + +def parse_subscriptions_page(html: str, *, base: str = _ROSTER_BASE) -> list[Membership]: + """Every membership on the account's /subscriptions page. + + Refuses rather than guessing, because every conclusion downstream is drawn + from ABSENCE — a roster that comes back short reads as "you cancelled + those". So this raises when: + + * the active card is missing — as SubscribeStarAuthError if the page is a + login or age wall (the fix is credentials), otherwise as drift; + * a row has no creator link or no numeric creator id; + * anything renders after a card's table, or the page carries a `page=` link. + Both cards are paginatable (`data-view="app#embed_pagination"`), and the + characterized account was too small to show what pagination looks like — + so possible pagination is treated as a roster FC cannot prove complete. + + A missing cancelled card is NOT drift: an account that has never cancelled + plausibly has no such table. A creator present in both tables is reported + once, as active — a current subscription is the fact that matters. + """ + active = _roster_table(html, _ROSTER_ACTIVE) + if active is None: + if any(marker in html for marker in _LOGIN_MARKERS): + raise SubscribeStarAuthError( + "SubscribeStar served a login/age wall instead of the " + "subscriptions page (cookies expired or age cookie missing)" + ) + raise SubscribeStarDriftError( + f"SubscribeStar subscriptions page has no {_ROSTER_ACTIVE!r} card " + f"— {_describe_page(html)}" + ) + roster_region = html[html.find(f'data-identifier="{_ROSTER_ACTIVE}"'):] + if _ROSTER_PAGE_LINK_RE.search(roster_region): + raise SubscribeStarDriftError( + "SubscribeStar subscriptions page carries a page= link — the roster " + "may be paginated, and FC cannot prove it is complete (note #3989)" + ) + + memberships: list[Membership] = [] + seen: set[str] = set() + for identifier, found in ( + (_ROSTER_ACTIVE, active), + (_ROSTER_CANCELLED, _roster_table(html, _ROSTER_CANCELLED)), + ): + if found is None: + continue + table, trailing = found + if trailing.strip(): + raise SubscribeStarDriftError( + f"SubscribeStar roster card {identifier!r} renders content after " + f"its table — possibly pagination, so the roster cannot be " + f"proven complete (note #3989)" + ) + for membership in _roster_rows(table, identifier, base): + if membership.campaign_id in seen: + continue + seen.add(membership.campaign_id) + memberships.append(membership) + return memberships + + class SubscribeStarClient: """Synchronous SubscribeStar HTML-scrape read client. Construct with a path to a Netscape cookies.txt (the same file CredentialService.get_cookies_path @@ -645,6 +846,23 @@ class SubscribeStarClient: return None return _extract_creator_name(html) + # -- membership roster (#387 D1) ---------------------------------------- + + def iter_memberships(self, user_id: str | None = None) -> Iterator[Membership]: + """Yield every subscription the account holds (note #3989). + + `user_id` exists for the seam's signature (note #3970) and is ignored: + the page is the logged-in account's own, so there is nothing to resolve. + The sweep only resolves an id for a client that exposes + `current_user_id`, which this one does not. + + One request, and the whole page is parsed before anything is yielded, so + a drift error can never leave a caller holding part of a roster. + """ + self._session.headers["Referer"] = f"{_ROSTER_BASE}/" + resp = self._get(_ROSTER_URL) + yield from parse_subscriptions_page(resp.text or "", base=_ROSTER_BASE) + # -- verify ------------------------------------------------------------ def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 10b01ec..6e2b47e 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -1131,3 +1131,199 @@ def vacuum_analyze() -> dict: done.append(table) log.info("vacuum_analyze complete: %s", done) return {"vacuumed": done} + + +@celery.task( + name="backend.app.tasks.maintenance.group_discord_drops", + soft_time_limit=1800, time_limit=2100, +) +def group_discord_drops() -> str: + """Milestone 388 E2: group Discord message-posts into the drops FC authors. + + Lives on the MAINTENANCE lane, not the ml lane, even though it reads SigLIP + vectors — it does no inference and imports no ML library, and the ml-worker + is an OPTIONAL container (B3). Routing it to 'ml' would silently disable + grouping on every stack that runs a GPU agent and drops that container, + which is the same trap gpu_queue.py was moved here to avoid. + + Async body under its own loop, per the _async_session contract: the sweep + needs pgvector column reads and the shared services are async. + """ + import asyncio + + from ..services.discord_grouping import sweep + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, engine = async_session_factory() + try: + async with async_factory() as session: + result = await sweep(session) + await session.commit() + return result + finally: + await engine.dispose() + + res = asyncio.run(_run()) + if not res["enabled"]: + return "disabled" + return ( + f"sources={res['sources']} created={res['posts_created']} " + f"joined={res['images_joined']}" + ) + + +@celery.task( + name="backend.app.tasks.maintenance.match_post_associations", + soft_time_limit=900, time_limit=1200, +) +def match_post_associations() -> str: + """Milestone 388 E5: propose which Patreon post announced which Discord drop. + + Proposes only — every pair lands in a review queue and nothing is linked + until the operator accepts. Maintenance lane for the same reason as the + grouper: no inference, no ML library, and it must not depend on the + optional ml-worker being present. + """ + import asyncio + + from ..services.post_association_service import rescan + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, engine = async_session_factory() + try: + async with async_factory() as session: + result = await rescan(session) + await session.commit() + return result + finally: + await engine.dispose() + + res = asyncio.run(_run()) + if not res["enabled"]: + return "disabled" + return f"scanned={res['scanned']} proposed={res['proposed']}" + + +# The wall-clock budget for ONE platform's roster walk. Rule 156 distinguishes +# this from the per-REQUEST timeout the client already has: a paginated roster +# behind an endpoint that answers every page slowly-but-within-timeout would +# never trip that one, and would sit on a worker indefinitely. This is the wait +# that bounds the whole walk. +MEMBERSHIP_SYNC_BUDGET_SECONDS = 240.0 + + +@celery.task( + name="backend.app.tasks.maintenance.sync_memberships", + soft_time_limit=900, time_limit=1200, +) +def sync_memberships() -> str: + """Milestone 387 C3: walk each platform's membership roster into the DB. + + Daily, because memberships change on a BILLING cycle rather than a download + cadence — polling an account-scoped endpoint more often would be both + pointless and less polite than the browser. + + Rule 89's four, and where each actually lives: + * recovery — `touch_membership` is an upsert and never deletes, so + recovery is simply the next run; a dead run leaves the previous roster + intact rather than a half-written one (`sync_platform` completes the + fetch before writing anything). + * retention — per C1, rows age out and are never deleted on + disappearance, because disappearing IS the signal C4 reads. + * wall-clock timeout — MEMBERSHIP_SYNC_BUDGET_SECONDS per platform, + plus the task's own soft/hard limits. + * duration tracking — the TaskRun celery-signal plumbing, same as every + other sweep here. + + Rule 164: this is the rule's own named exception — a genuinely external + feature that may call out. It never gates startup, and a failure leaves a + VISIBLE stale state (membership_sync) rather than an empty roster that + reads as "you subscribe to nothing". + """ + import asyncio + + from ..services.artist_membership_service import rescan as membership_rescan + from ..services.credential_crypto import CredentialCrypto + from ..services.credential_service import CredentialService + from ..services.membership_roster import roster_user_id, sync_platform + from ..services.patreon_client import PatreonClient + from ..services.subscribestar_client import SubscribeStarClient + from ._async_session import async_session_factory + + key_path = IMAGES_ROOT / "secrets" / "credential_key.b64" + + # platform -> how to build a client from a cookies path. A platform is in + # the sweep only if it is here AND its client exposes `iter_memberships` + # AND a credential exists — three independent gates, each silent, so a + # platform is added with one line here and nothing else. SubscribeStar (D1) + # was the second; the only other change it needed was `roster_user_id` + # replacing an unconditional Patreon-only call below. + builders = {"patreon": PatreonClient, "subscribestar": SubscribeStarClient} + + async def _run() -> dict: + async_factory, engine = async_session_factory() + results = [] + try: + for platform, build in builders.items(): + async with async_factory() as session: + cred = CredentialService(session, CredentialCrypto(key_path)) + cookies = await cred.get_cookies_path(platform) + if cookies is None: + # No credential is not an error — the operator simply has + # not connected this platform. Recording a failure here + # would light up the UI for a feature they never enabled. + results.append({"platform": platform, "skipped": "no credential"}) + continue + + client = build(str(cookies)) + if getattr(client, "iter_memberships", None) is None: + # The seam, probed not required (#387 C2). A client without + # it makes the feature invisible for that platform — no + # flag, no config row, no "unsupported" branch. + results.append({"platform": platform, "skipped": "no seam"}) + continue + + async def fetch(_client=client): + # The client is sync (`requests`); run it off the loop so a + # slow roster does not block the event loop, and bound the + # whole walk rather than only its individual requests. + def _walk(): + return list(_client.iter_memberships(roster_user_id(_client))) + + return await asyncio.wait_for( + asyncio.to_thread(_walk), + timeout=MEMBERSHIP_SYNC_BUDGET_SECONDS, + ) + + async with async_factory() as session: + results.append( + await sync_platform(session, platform=platform, fetch=fetch) + ) + + # #388 E4: offer the freshly-synced roster to the artists FC already + # tracks. Chained here rather than given its own beat entry because + # a suggestion can only be as good as the roster behind it — running + # it on any other cadence would just propose from staler data. + suggested = None + if any(r.get("ok") for r in results): + async with async_factory() as session: + suggested = (await membership_rescan(session))["proposed"] + await session.commit() + return {"results": results, "suggested": suggested} + finally: + await engine.dispose() + + res = asyncio.run(_run()) + parts = [] + for r in res["results"]: + if "skipped" in r: + parts.append(f"{r['platform']}=skipped({r['skipped']})") + elif r.get("ok"): + parts.append(f"{r['platform']}={r['count']}") + else: + parts.append(f"{r['platform']}=FAILED({r['error']})") + if res.get("suggested") is not None: + parts.append(f"suggested={res['suggested']}") + return " ".join(parts) or "no platforms" diff --git a/docker-compose.yml b/docker-compose.yml index 0bca4f2..e7d919d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ # explicitly skips the override and pulls the published :latest images. # # FabledCurator has no authentication. Whatever can reach ${PORT} is an -# administrator, including over the stored Patreon/SubscribeStar/Pixiv session +# administrator, including over the stored Patreon/SubscribeStar session # cookies. Do not publish this port beyond a network you trust — see # "Before you expose it" in README.md. diff --git a/extension/background/background.js b/extension/background/background.js index edbc2a0..dd5155a 100644 --- a/extension/background/background.js +++ b/extension/background/background.js @@ -1,32 +1,35 @@ /** - * Background script — message router + Discord token capture - * (webRequest) + Pixiv PKCE OAuth. Direct port of GS background.js; - * api.js client points at FC instead of GS. + * Background script — message router + Discord token capture (webRequest). + * Direct port of GS background.js; api.js client points at FC instead of GS. + * + * pixiv's PKCE OAuth flow lived here until FC retired pixiv (milestone #406). + * It was removed together with pixiv's host permissions rather than left + * behind: a webRequest listener on a host the manifest no longer grants is at + * best dead and at worst a startup failure for the whole background script. */ let discordToken = null; let discordTokenCapturedAt = null; -let pixivRefreshToken = null; -let pixivTokenCapturedAt = null; -let pixivOAuthPending = null; - -const PIXIV_CLIENT_ID = 'MOBrBDS8blbauoSck0ZfDbtuzpyT'; -const PIXIV_CLIENT_SECRET = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj'; -const PIXIV_OAUTH_URL = 'https://app-api.pixiv.net/web/v1/login'; -const PIXIV_TOKEN_URL = 'https://oauth.secure.pixiv.net/auth/token'; -const PIXIV_REDIRECT_URI = 'https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback'; - let initialized = false; async function ensureInitialized() { if (initialized) return; await api.init(); await loadDiscordToken(); - await loadPixivToken(); + await forgetRetiredPixivToken(); initialized = true; } +// A browser that authenticated pixiv before the retirement still holds a live +// OAuth refresh token in extension storage. Nothing reads it any more, and a +// credential for a service FC no longer uses is a liability with no benefit +// (the same reasoning as the server-side cleanup, issue #3980). Removing keys +// that are absent is a no-op, so this is safe on every startup. +async function forgetRetiredPixivToken() { + await browser.storage.local.remove(['pixivRefreshToken', 'pixivTokenCapturedAt']); +} + browser.runtime.onInstalled.addListener(() => ensureInitialized()); browser.runtime.onStartup.addListener(() => ensureInitialized()); ensureInitialized().catch(e => console.error('init failed:', e)); @@ -141,98 +144,6 @@ async function saveDiscordToken(token) { await browser.storage.local.set({ discordToken: token, discordTokenCapturedAt }); } -// ---- Pixiv PKCE OAuth ---- - -async function loadPixivToken() { - const s = await browser.storage.local.get(['pixivRefreshToken', 'pixivTokenCapturedAt']); - pixivRefreshToken = s.pixivRefreshToken || null; - pixivTokenCapturedAt = s.pixivTokenCapturedAt || null; -} - -async function savePixivToken(token) { - pixivRefreshToken = token; - pixivTokenCapturedAt = new Date().toISOString(); - await browser.storage.local.set({ pixivRefreshToken: token, pixivTokenCapturedAt }); -} - -function generateCodeVerifier() { - const a = new Uint8Array(32); - crypto.getRandomValues(a); - return base64UrlEncode(a); -} - -async function generateCodeChallenge(verifier) { - const data = new TextEncoder().encode(verifier); - const hash = await crypto.subtle.digest('SHA-256', data); - return base64UrlEncode(new Uint8Array(hash)); -} - -function base64UrlEncode(buf) { - return btoa(String.fromCharCode(...buf)).replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, ''); -} - -async function initiatePixivOAuth() { - const codeVerifier = generateCodeVerifier(); - const codeChallenge = await generateCodeChallenge(codeVerifier); - - const params = new URLSearchParams({ - code_challenge: codeChallenge, - code_challenge_method: 'S256', - client: 'pixiv-android', - }); - const tab = await browser.tabs.create({ url: `${PIXIV_OAUTH_URL}?${params}` }); - - return new Promise((resolve, reject) => { - pixivOAuthPending = { codeVerifier, tabId: tab.id, resolve, reject }; - setTimeout(() => { - if (pixivOAuthPending && pixivOAuthPending.tabId === tab.id) { - pixivOAuthPending = null; - reject(new Error('Pixiv OAuth timed out (5 min)')); - } - }, 5 * 60 * 1000); - }); -} - -browser.webRequest.onBeforeRedirect.addListener( - async (details) => { - if (!pixivOAuthPending) return; - if (details.tabId !== pixivOAuthPending.tabId) return; - const url = new URL(details.redirectUrl); - const code = url.searchParams.get('code'); - if (!code) return; - const verifier = pixivOAuthPending.codeVerifier; - const resolve = pixivOAuthPending.resolve; - const reject = pixivOAuthPending.reject; - pixivOAuthPending = null; - try { - const tokenResp = await fetch(PIXIV_TOKEN_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams({ - client_id: PIXIV_CLIENT_ID, - client_secret: PIXIV_CLIENT_SECRET, - code, - code_verifier: verifier, - grant_type: 'authorization_code', - include_policy: 'true', - redirect_uri: PIXIV_REDIRECT_URI, - }), - }); - const body = await tokenResp.json(); - if (!body.refresh_token) { - reject(new Error(`Pixiv token exchange failed: ${JSON.stringify(body)}`)); - return; - } - await savePixivToken(body.refresh_token); - try { await browser.tabs.remove(details.tabId); } catch {} - resolve(body.refresh_token); - } catch (e) { - reject(e); - } - }, - { urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] }, -); - // Extract → verify → upload one cookie-auth platform. Returns a structured // outcome so the two callers (EXPORT_COOKIES single, EXPORT_ALL_COOKIES) shape // their own response + skip semantics. Verifies the captured cookies are @@ -277,8 +188,6 @@ browser.runtime.onMessage.addListener(async (msg) => { } } else if (key === 'discord') { status[key] = { hasToken: !!discordToken, capturedAt: discordTokenCapturedAt }; - } else if (key === 'pixiv') { - status[key] = { hasToken: !!pixivRefreshToken, capturedAt: pixivTokenCapturedAt }; } else { status[key] = {}; } @@ -306,13 +215,6 @@ browser.runtime.onMessage.addListener(async (msg) => { await api.uploadCredentials('discord', 'token', discordToken); return { success: true }; } - if (key === 'pixiv') { - if (!pixivRefreshToken) { - await initiatePixivOAuth(); - } - await api.uploadCredentials('pixiv', 'token', pixivRefreshToken); - return { success: true }; - } return { error: 'Unsupported platform.' }; } catch (e) { return { error: e.message }; diff --git a/extension/lib/platforms.js b/extension/lib/platforms.js index c3f0c5a..86972d9 100644 --- a/extension/lib/platforms.js +++ b/extension/lib/platforms.js @@ -60,14 +60,6 @@ const PLATFORMS = { urlPattern: /^https?:\/\/(www\.)?discord\.com/, note: 'Open Discord in browser to capture token', }, - pixiv: { - name: 'Pixiv', - domains: ['.pixiv.net', 'www.pixiv.net', 'pixiv.net'], - authType: 'token', - color: '#0096FA', - urlPattern: /^https?:\/\/(www\.)?pixiv\.net/, - note: 'Click to authenticate via OAuth', - }, }; /** @@ -88,7 +80,6 @@ const PLATFORM_ARTIST_PATTERNS = { patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i, subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i, hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i, - pixiv: /^https?:\/\/(www\.)?pixiv\.net\/(en\/)?users\/\d+/i, }; function getPlatformFromUrl(url) { diff --git a/extension/manifest.json b/extension/manifest.json index b488988..3227775 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -32,9 +32,6 @@ "*://*.subscribestar.adult/*", "*://*.hentai-foundry.com/*", "*://*.discord.com/*", - "*://*.pixiv.net/*", - "*://app-api.pixiv.net/*", - "*://oauth.secure.pixiv.net/*", "*://*/*" ], @@ -59,8 +56,7 @@ "*://*.patreon.com/*", "*://*.subscribestar.com/*", "*://*.subscribestar.adult/*", - "*://*.hentai-foundry.com/*", - "*://*.pixiv.net/*" + "*://*.hentai-foundry.com/*" ], "js": ["lib/platforms.js", "content/content-script.js"], "css": ["content/content-script.css"], diff --git a/extension/popup/popup.js b/extension/popup/popup.js index 771bb6a..7becdba 100644 --- a/extension/popup/popup.js +++ b/extension/popup/popup.js @@ -108,7 +108,7 @@ function createPlatformCard(key, platform, status) { card.className = 'platform-card'; card.dataset.platform = key; - const isTokenOnly = platform.authType === 'token' && !['discord', 'pixiv'].includes(key); + const isTokenOnly = platform.authType === 'token' && key !== 'discord'; const discordNeedsToken = key === 'discord' && !status.hasToken; if (isTokenOnly || discordNeedsToken) card.classList.add('disabled'); @@ -141,7 +141,6 @@ function createPlatformCard(key, platform, status) { function statusText(s, platform, key) { if (key === 'discord') return s.hasToken ? 'Token captured — ready' : 'Open Discord to capture token'; - if (key === 'pixiv') return s.hasToken ? 'Token captured — ready' : 'Click to authenticate via OAuth'; if (platform.authType === 'token') return 'Manual token entry required'; if (s.error) return 'Error checking cookies'; if (!s.hasCookies || !s.cookieCount) return 'No cookies — log in first'; @@ -149,7 +148,6 @@ function statusText(s, platform, key) { } function statusClass(s, platform, key) { if (key === 'discord') return s.hasToken ? 'ready' : 'no-cookies'; - if (key === 'pixiv') return s.hasToken ? 'ready' : 'no-cookies'; if (platform.authType === 'token') return 'no-cookies'; if (s.error) return 'error'; if (!s.hasCookies || !s.cookieCount) return 'no-cookies'; diff --git a/extension/test/platforms.spec.js b/extension/test/platforms.spec.js index df1e53a..f869751 100644 --- a/extension/test/platforms.spec.js +++ b/extension/test/platforms.spec.js @@ -18,7 +18,6 @@ describe('getPlatformFromUrl', () => { expect(getPlatformFromUrl('https://subscribestar.adult/someone')).toBe('subscribestar') expect(getPlatformFromUrl('https://www.hentai-foundry.com/user/someone')).toBe('hentaifoundry') expect(getPlatformFromUrl('https://discord.com/channels/@me')).toBe('discord') - expect(getPlatformFromUrl('https://www.pixiv.net/en/users/123')).toBe('pixiv') }) it('accepts http as well as https, with or without www', () => { @@ -32,6 +31,15 @@ describe('getPlatformFromUrl', () => { expect(getPlatformFromUrl('')).toBe(null) }) + it('returns null for pixiv, retired at milestone #406', () => { + // Retired on the operator's platform-focus decision (rule #171). Same guard + // as deviantart's below, and for the same reason: an absence nothing asserts + // is an absence a later edit can quietly undo. + expect(getPlatformFromUrl('https://www.pixiv.net/en/users/12345')).toBe(null) + expect(PLATFORMS.pixiv).toBeUndefined() + expect(PLATFORM_ARTIST_PATTERNS.pixiv).toBeUndefined() + }) + it('returns null for deviantart, retired at #3069', () => { // The 2026-07-05 product decision (FC downloaders = art-dedicated services // only) left deviantart wired for seven weeks. Asserting the negative is @@ -80,12 +88,6 @@ describe('isArtistPage', () => { ) }) - it('matches Pixiv numeric user pages, with or without the /en/ prefix', () => { - expect(isArtistPage('https://www.pixiv.net/users/12345', 'pixiv')).toBe(true) - expect(isArtistPage('https://www.pixiv.net/en/users/12345', 'pixiv')).toBe(true) - expect(isArtistPage('https://www.pixiv.net/en/artworks/999', 'pixiv')).toBe(false) - }) - it('returns false for a platform with no artist pattern (discord)', () => { expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false) }) @@ -123,8 +125,7 @@ describe('platform table integrity', () => { const samples = { patreon: 'https://www.patreon.com/cw/Atole', subscribestar: 'https://subscribestar.adult/someone', - hentaifoundry: 'https://www.hentai-foundry.com/user/someone', - pixiv: 'https://www.pixiv.net/en/users/12345' + hentaifoundry: 'https://www.hentai-foundry.com/user/someone' } for (const [key, url] of Object.entries(samples)) { expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true) @@ -179,8 +180,9 @@ describe('manifest.json agrees with the platform table', () => { for (const h of manifest.host_permissions) { if (h === '*://*/*') continue const host = hostOf(h) - // pixiv's OAuth/API hosts are pixiv infrastructure, not creator pages, - // so they are matched by suffix rather than by the domains list. + // Suffix matching lets a platform's infrastructure subdomains belong to + // it without listing each one. (It was added for pixiv's OAuth hosts, + // which left with pixiv at milestone #406; the rule itself is general.) const claimed = Object.values(PLATFORMS).some( (p) => p.domains.includes(host) || p.domains.some((d) => host.endsWith(d)) ) diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg index 6b07fb4..9cb1c69 100644 --- a/frontend/public/favicon.svg +++ b/frontend/public/favicon.svg @@ -1,5 +1,24 @@ - + + FabledCurator + - + + diff --git a/frontend/public/logo.svg b/frontend/public/logo.svg new file mode 100644 index 0000000..ff3db36 --- /dev/null +++ b/frontend/public/logo.svg @@ -0,0 +1,165 @@ + + FabledCurator + + + + + + + + \ No newline at end of file diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index 8b99c2b..31a9645 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -6,7 +6,8 @@ FabledCurator {{ health.icon }} @@ -282,9 +283,9 @@ onUnmounted(() => { if (healthTimer) clearInterval(healthTimer) }) display: flex; align-items: center; flex-shrink: 0; - /* A RouterLink since milestone 365 — it is the path to /system, not just an - indicator. Reset the anchor so turning a span into a link changed nothing - about how the nav reads. */ + /* A RouterLink since milestone 365 — it is the path to the Settings System + tab, not just an indicator. Reset the anchor so turning a span into a + link changed nothing about how the nav reads. */ text-decoration: none; color: inherit; border-radius: 50%; diff --git a/frontend/src/components/posts/FeedEmptyState.vue b/frontend/src/components/posts/FeedEmptyState.vue new file mode 100644 index 0000000..873b058 --- /dev/null +++ b/frontend/src/components/posts/FeedEmptyState.vue @@ -0,0 +1,296 @@ + + + + + diff --git a/frontend/src/components/posts/FeedStatusRibbon.vue b/frontend/src/components/posts/FeedStatusRibbon.vue new file mode 100644 index 0000000..9addca0 --- /dev/null +++ b/frontend/src/components/posts/FeedStatusRibbon.vue @@ -0,0 +1,80 @@ + + + + + diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index f595a70..f3b7216 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -6,6 +6,19 @@ {{ post.source?.platform ?? 'filesystem import' }} + + + + grouped by FabledCurator + · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }} + + + · updated {{ grewRelative }} +

{{ displayTitle }}

+ +

{{ synthesisTitle }}

Post {{ post.external_post_id }}

@@ -102,6 +128,25 @@ @click="toggleDesc" >{{ descExpanded ? 'Show less' : 'Show more' }} + + +
+ + + {{ a.role === 'announces' + ? 'The full set is in Discord' : 'Announced on Patreon' }} + +
+ @@ -163,6 +208,18 @@ const images = computed(() => props.post.thumbnails || []) const totalImages = computed(() => images.value.length + (props.post.thumbnails_more || 0)) const plainTitle = computed(() => toPlainText(props.post.post_title)) +// #388 E2. Non-null `synthesized_by` means FC authored this row by grouping a +// creator's drop; `synthesis` carries what it was built from. Read defensively +// — a post fetched before the field existed, or any surface that composes a +// post dict by hand, must degrade to "not synthetic" rather than throw. +const synthesized = computed(() => Boolean(props.post.synthesized_by)) +const messageCount = computed(() => props.post.synthesis?.message_count ?? 0) +const synthesisTitle = computed(() => { + const n = messageCount.value + if (!n) return 'Grouped from Discord' + return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}` +}) + const hero = computed(() => images.value[0]) // The thumbnail strip spans the hero's full width (CSS grid, equal columns), // rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are @@ -189,16 +246,29 @@ const moreCount = computed(() => { const railCols = computed(() => rail.value.length + (moreCount.value > 0 ? 1 : 0)) const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at) + +// #388 E3. A grouping stays OPEN, so its own date and its latest activity are +// different facts. The card keeps showing when the drop STARTED — that is the +// post's identity — and reports growth separately, because "this post is from +// Tuesday but gained images this morning" is the whole signal that chat +// content is trickling in. +const grewAt = computed(() => (synthesized.value ? props.post.last_grew_at : null)) +// #388 E5. Accepted links only — a pending proposal lives in the review queue, +// never beside the artwork. Defaults to [] so a post dict from before the +// feature (or composed by hand) renders without a link rather than throwing. +const associations = computed(() => props.post.associations || []) +const grewRelative = computed(() => (grewAt.value ? relativeFrom(grewAt.value) : '')) const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString()) -const relativeDate = computed(() => { - const then = new Date(sortDateIso.value).getTime() +function relativeFrom (iso) { + const then = new Date(iso).getTime() const diff = (Date.now() - then) / 1000 if (diff < 60) return `${Math.floor(diff)}s ago` if (diff < 3600) return `${Math.floor(diff / 60)}m ago` if (diff < 86400) return `${Math.floor(diff / 3600)}h ago` if (diff < 86400 * 30) return `${Math.floor(diff / 86400)}d ago` - return new Date(sortDateIso.value).toLocaleDateString() -}) + return new Date(iso).toLocaleDateString() +} +const relativeDate = computed(() => relativeFrom(sortDateIso.value)) // --- images → post-scoped modal --------------------------------------- async function fullImageIds () { @@ -337,6 +407,27 @@ function formatBytes (n) { font-weight: 600; } .fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); } + +/* Quiet, not decorative: the marker has to be legible on every card without + turning a synthetic post into the loudest thing in the feed. */ +.fc-post-card__synthetic { + color: rgb(var(--v-theme-on-surface-variant)); +} + +/* Growth is news, so it gets the accent the rest of the meta line doesn't — + but it is still the meta line, not a badge competing with the artwork. */ +.fc-post-card__grew { color: rgb(var(--v-theme-accent)); } + +.fc-post-card__assoc { margin-top: 8px; } +.fc-post-card__assoc-link { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 0.8125rem; + color: rgb(var(--v-theme-accent)); + text-decoration: none; +} +.fc-post-card__assoc-link:hover { text-decoration: underline; } .fc-post-card__date, .fc-post-card__meta { white-space: nowrap; } diff --git a/frontend/src/components/settings/BrowserExtensionCard.vue b/frontend/src/components/settings/BrowserExtensionCard.vue index 4226eba..50b6a6a 100644 --- a/frontend/src/components/settings/BrowserExtensionCard.vue +++ b/frontend/src/components/settings/BrowserExtensionCard.vue @@ -19,7 +19,7 @@

Pushes session cookies from supported platforms - (patreon, subscribestar, hentaifoundry, discord, pixiv) + (patreon, subscribestar, hentaifoundry, discord) into FabledCurator, and lets you add a creator as a source from their page in one click.

diff --git a/frontend/src/components/settings/DiscordGroupingCard.vue b/frontend/src/components/settings/DiscordGroupingCard.vue new file mode 100644 index 0000000..c7a91d1 --- /dev/null +++ b/frontend/src/components/settings/DiscordGroupingCard.vue @@ -0,0 +1,134 @@ + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 372f920..d0b3773 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -14,6 +14,10 @@
+ + + +
@@ -80,6 +84,10 @@ import DbMaintenanceCard from './DbMaintenanceCard.vue' import VideoEmbeddingCard from './VideoEmbeddingCard.vue' import CropProposersCard from './CropProposersCard.vue' import HeadsCard from './HeadsCard.vue' +import DiscordGroupingCard from './DiscordGroupingCard.vue' +import MembershipRosterCard from './MembershipRosterCard.vue' +import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue' +import PostAssociationsCard from './PostAssociationsCard.vue' import GpuAgentCard from './GpuAgentCard.vue' import AliasTable from './AliasTable.vue' import BackupCard from './BackupCard.vue' diff --git a/frontend/src/components/settings/MembershipRosterCard.vue b/frontend/src/components/settings/MembershipRosterCard.vue new file mode 100644 index 0000000..1052c01 --- /dev/null +++ b/frontend/src/components/settings/MembershipRosterCard.vue @@ -0,0 +1,97 @@ + + + + + diff --git a/frontend/src/components/settings/MembershipSuggestionsCard.vue b/frontend/src/components/settings/MembershipSuggestionsCard.vue new file mode 100644 index 0000000..a2ad797 --- /dev/null +++ b/frontend/src/components/settings/MembershipSuggestionsCard.vue @@ -0,0 +1,74 @@ + + + + + diff --git a/frontend/src/components/settings/PostAssociationsCard.vue b/frontend/src/components/settings/PostAssociationsCard.vue new file mode 100644 index 0000000..141d789 --- /dev/null +++ b/frontend/src/components/settings/PostAssociationsCard.vue @@ -0,0 +1,128 @@ + + + + + diff --git a/frontend/src/views/SystemView.vue b/frontend/src/components/settings/SystemHealthTab.vue similarity index 80% rename from frontend/src/views/SystemView.vue rename to frontend/src/components/settings/SystemHealthTab.vue index 18ead8a..c1853aa 100644 --- a/frontend/src/views/SystemView.vue +++ b/frontend/src/components/settings/SystemHealthTab.vue @@ -1,7 +1,10 @@ diff --git a/frontend/src/components/subscriptions/SchedulerStatusBar.vue b/frontend/src/components/subscriptions/SchedulerStatusBar.vue index 59f8bb2..464fce1 100644 --- a/frontend/src/components/subscriptions/SchedulerStatusBar.vue +++ b/frontend/src/components/subscriptions/SchedulerStatusBar.vue @@ -27,7 +27,10 @@ import { computed } from 'vue' import { formatRelative } from '../../utils/date.js' const props = defineProps({ - // { last_tick_at, next_due_at, due_now, auto_sources } | null + // { last_tick_at, next_due_at, due_now, auto_sources, + // failing_sources, no_access_sources, platform_cooldowns } | null + // This bar renders the scheduling half; the ingestion counts are read by the + // front door's FeedStatusRibbon (#387 B3) off the same payload. status: { type: Object, default: null }, }) diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 5c87393..e026aed 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -77,7 +77,7 @@ const recapturing = computed(() => !!props.source.backfill_recapture) // Recover / recapture are native-ingester features (ledger-bypass re-walk and // post-text re-grab), available to every native platform — not just Patreon. // Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. -const NATIVE_PLATFORMS = ['patreon', 'subscribestar', 'pixiv'] +const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) diff --git a/frontend/src/components/subscriptions/SourceHealthDot.vue b/frontend/src/components/subscriptions/SourceHealthDot.vue index 145044d..304d564 100644 --- a/frontend/src/components/subscriptions/SourceHealthDot.vue +++ b/frontend/src/components/subscriptions/SourceHealthDot.vue @@ -10,6 +10,10 @@
Last checked: {{ lastCheckedText }}
Next check: {{ nextCheckText }}
+
{{ noAccessText }}
+
+ {{ noAccessReason }} +
Failures: {{ source.consecutive_failures }}
@@ -29,14 +33,42 @@ const props = defineProps({ warningThreshold: { type: Number, default: 5 }, }) +const noAccess = computed(() => props.source.error_type === 'tier_limited') + const level = computed(() => { if (!props.source.last_checked_at) return 'unchecked' const f = props.source.consecutive_failures || 0 - if (f === 0) return 'healthy' + // No-access outranks 'healthy' but is NOT a failure grade: the walk worked, + // the content simply isn't ours. Checked after failures so a source that is + // genuinely erroring still reads as erroring. + if (f === 0) return noAccess.value ? 'no-access' : 'healthy' if (f < props.warningThreshold) return 'warning' return 'critical' }) +// The count comes from the last walk's run_stats and is only joined in by the +// list endpoint, so it can legitimately be absent — say the state without it +// rather than printing a fabricated zero. +const noAccessText = computed(() => { + const n = props.source.tier_gated_count + return n + ? `${n} post${n === 1 ? '' : 's'} you don't have access to` + : "Some posts are behind a tier you don't hold" +}) + +// #387 C5: the learned roster's explanation for the line above, when it has +// one. The backend sends null for every case where the roster is not evidence — +// campaign absent, roster stale, never swept, status not yet characterised — so +// there is deliberately NO fallback sentence here. A default would turn "we +// don't know why" into a reason, which is the one thing this step must not do. +const GATED_REASONS = { + lapsed: "You're not a patron any more — resubscribe, or disable this source.", + tier: "Your tier doesn't cover these posts — upgrade, or leave them be.", + free: "You follow this creator for free — these posts are for paying patrons.", +} + +const noAccessReason = computed(() => GATED_REASONS[props.source.gated_reason] || null) + const ariaLabel = computed(() => `source health: ${level.value}`) const lastCheckedText = computed(() => formatRelative(props.source.last_checked_at)) @@ -63,6 +95,9 @@ const truncatedError = computed(() => { } .fc-health-dot--unchecked { background-color: rgb(var(--v-theme-on-surface-variant)); opacity: 0.5; } .fc-health-dot--healthy { background-color: rgb(var(--v-theme-success, 76 175 80)); } +/* Matches the 'info' severity FailingSourcesCard already assigns tier_limited — + deliberately not a warning/error hue: nothing is broken. */ +.fc-health-dot--no-access { background-color: rgb(var(--v-theme-info, 33 150 243)); } .fc-health-dot--warning { background-color: rgb(var(--v-theme-warning, 255 167 38)); } .fc-health-dot--critical { background-color: rgb(var(--v-theme-error, 244 67 54)); } @@ -70,6 +105,15 @@ const truncatedError = computed(() => { font-size: 0.85rem; line-height: 1.4; } +.fc-health-tip__gated { + color: rgb(var(--v-theme-info, 33 150 243)); +} +/* The reason is subordinate to the count it explains: same block, quieter, so + a tooltip that has one does not read as two separate findings. */ +.fc-health-tip__why { + color: rgb(var(--v-theme-on-surface-variant)); + max-width: 24rem; +} .fc-health-tip__err { margin-top: 0.25rem; color: rgb(var(--v-theme-error, 244 67 54)); diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 10de1b0..0c287a0 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -48,6 +48,20 @@ {{ source.last_error }} + + {{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }} + + {{ noAccessTip }} + +