diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index f56f4ba..5835341 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -81,8 +81,18 @@ jobs: set -eux # Same service-IP dance as build.yml's integration job; see the long # comment there for why the job name must stay separator-free. - PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - test -n "$PG" + # Only THIS job's services. The runner's docker daemon is shared, so another + # repo's job can be running beside this one, and a job-name filter matches its + # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's + # Postgres). act names every container of a task GITEA-ACTIONS-TASK--...: read + # from our own container (its id is in the /etc/hostname bind mount) and + # require exactly one match. + SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true) + SELF=${SELF:-$(hostname)} + TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-') + test -n "$TASK" + PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q) + test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1 PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") test -n "$PG_IP" echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV" diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 2dab36a..6bedc66 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -532,9 +532,20 @@ jobs: echo "=== container landscape (diagnostic for filter scoping) ===" docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' echo "=== end landscape ===" - PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" + # Only THIS job's services. The runner's docker daemon is shared, so another + # repo's job can be running beside this one, and a job-name filter matches its + # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's + # Postgres). act names every container of a task GITEA-ACTIONS-TASK--...: read + # from our own container (its id is in the /etc/hostname bind mount) and + # require exactly one match. + SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true) + SELF=${SELF:-$(hostname)} + TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-') + test -n "$TASK" + PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q) + test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1 + RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q) + test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1 PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") test -n "$PG_IP" && test -n "$RD_IP" @@ -776,6 +787,15 @@ jobs: print("guard: no ext-* release yet — nothing to go backwards from") raise SystemExit(0) + # A version that is already signed is reused from its release, not + # signed again, so it cannot be a downgrade. The weekly refresh builds + # main, whose version trails whatever dev has signed since; refusing + # it here would stop :latest from ever being refreshed while dev is + # ahead. + if any(v == derived for v, _ in signed): + print(f"guard: {derived_s} is already signed — reused, not re-signed") + raise SystemExit(0) + hi, hi_tag = signed[-1] print(f"guard: derived={derived_s} highest already signed={hi_tag}") if derived < hi: @@ -1718,9 +1738,20 @@ jobs: # Service discovery mirrors the integration lane above: these jobs run # in a container against a mounted docker socket, so the services are # SIBLINGS reachable by IP, not by hostname. - PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" + # Only THIS job's services. The runner's docker daemon is shared, so another + # repo's job can be running beside this one, and a job-name filter matches its + # containers too (Steward run 8358 and Inkwell run 8653 each took another repo's + # Postgres). act names every container of a task GITEA-ACTIONS-TASK--...: read + # from our own container (its id is in the /etc/hostname bind mount) and + # require exactly one match. + SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true) + SELF=${SELF:-$(hostname)} + TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-') + test -n "$TASK" + PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q) + test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1 + RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q) + test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1 PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") test -n "$PG_IP" && test -n "$RD_IP" diff --git a/alembic/versions/0115_source_display_name.py b/alembic/versions/0115_source_display_name.py new file mode 100644 index 0000000..2f113df --- /dev/null +++ b/alembic/versions/0115_source_display_name.py @@ -0,0 +1,27 @@ +"""source.display_name — what a source is called on its platform. + +#4481. A Discord source's URL is a server id and a channel id, so the +Subscriptions page could only show two numbers. The Discord ingester already +loads the server and channel names to walk them; it now keeps them here, +refreshed on every walk. NULL until a walk has read it. + +Revision ID: 0115 +Revises: 0114 +Create Date: 2026-09-28 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0115" +down_revision = "0114" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("source", "display_name") diff --git a/alembic/versions/0116_single_color_threshold.py b/alembic/versions/0116_single_color_threshold.py new file mode 100644 index 0000000..915761d --- /dev/null +++ b/alembic/versions/0116_single_color_threshold.py @@ -0,0 +1,34 @@ +"""The single-color filter's default becomes near-total: 0.95 → 0.995. + +#4483. At 0.95 the filter rejected line art: a pencil doodle on white is +mostly white, and five of Todding's Discord doodles were skipped on import as +"single color", leaving their posts with text and no image. The predicate now +samples real pixels instead of a blurred thumbnail, and the default only +calls an image blank when it essentially is one. + +A stored 0.95 is the old default, so it moves with it. Any other value was +chosen by hand and is left alone. + +Revision ID: 0116 +Revises: 0115 +Create Date: 2026-09-28 + +""" +from alembic import op + +revision = "0116" +down_revision = "0115" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column("import_settings", "single_color_threshold", server_default="0.995") + op.execute( + "UPDATE import_settings SET single_color_threshold = 0.995 " + "WHERE single_color_threshold = 0.95" + ) + + +def downgrade() -> None: + op.alter_column("import_settings", "single_color_threshold", server_default="0.95") diff --git a/backend/app/api/extension.py b/backend/app/api/extension.py index 56b3078..bff2c8e 100644 --- a/backend/app/api/extension.py +++ b/backend/app/api/extension.py @@ -19,8 +19,10 @@ from ..models import AppSetting from ..services.extension_service import ( ExtensionService, InvalidUrlError, + PosterLookupError, UnknownArtistError, UnknownPlatformError, + UnknownSourceError, ) from ..services.source_service import KNOWN_PLATFORMS from ._responses import error_response as _bad @@ -121,6 +123,10 @@ async def quick_add_source(): # Patreon is canon: adding a Patreon source to an existing artist can take # the creator's Patreon display name (name only; the slug never moves). use_platform_name = body.get("use_platform_name") is True + # #4491: a Discord add can carry the posters picked in the panel. + discord_authors = body.get("discord_authors") + if discord_authors is not None and not _valid_authors(discord_authors): + return _bad("invalid_body", detail="discord_authors must be a list of {id, name}") from .credentials import _get_crypto @@ -133,6 +139,7 @@ async def quick_add_source(): result = await ExtensionService(session, _get_crypto()).quick_add_source( url, artist_id=artist_id, artist_name=artist_name, use_platform_name=use_platform_name, + discord_authors=discord_authors, ) except UnknownArtistError as exc: return _bad("not_found", detail=str(exc), status=404) @@ -147,6 +154,59 @@ async def quick_add_source(): return jsonify(result), (201 if result["created_source"] else 200) +def _valid_authors(value) -> bool: + """[{id, name}]: the id is what the list keys on, the name only shows it.""" + return isinstance(value, list) and all( + isinstance(a, dict) and isinstance(a.get("id"), str | int) + and not isinstance(a.get("id"), bool) + and (a.get("name") is None or isinstance(a.get("name"), str)) + for a in value + ) + + +@extension_bp.route("/discord/posters", methods=["GET"]) +async def discord_posters(): + """Who posts in the Discord channel `url` shows, lately, with the likely + creator marked (#4491). Reads the channel with the stored token; writes + nothing.""" + url = (request.args.get("url") or "").strip() + if not url: + return _bad("invalid_body", detail="url query parameter is required") + from .credentials import _get_crypto + + async with get_session() as session: + if not await _ext_key_required(session): + return _bad("unauthorized", status=401) + try: + result = await ExtensionService(session, _get_crypto()).discord_posters(url) + except (InvalidUrlError, UnknownPlatformError) as exc: + return _bad("invalid_url", detail=str(exc)) + except PosterLookupError as exc: + return _bad("posters_unavailable", detail=str(exc), status=409) + return jsonify(result) + + +@extension_bp.route("/discord/posters", methods=["POST"]) +async def set_discord_posters(): + """Set the poster list of the source following `url`'s channel.""" + body = await request.get_json(silent=True) + if not isinstance(body, dict) or not isinstance(body.get("url"), str): + return _bad("invalid_body", detail="url is required") + authors = body.get("authors") + if not _valid_authors(authors): + return _bad("invalid_body", detail="authors must be a list of {id, name}") + async with get_session() as session: + if not await _ext_key_required(session): + return _bad("unauthorized", status=401) + try: + result = await ExtensionService(session).set_discord_posters(body["url"], authors) + except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc: + return _bad("invalid_url", detail=str(exc)) + except UnknownSourceError as exc: + return _bad("not_found", detail=str(exc), status=404) + return jsonify(result) + + def _read_manifest_sync() -> dict | None: """All the filesystem-touching work for /api/extension/manifest, in a sync helper so the async route can dispatch it via diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 6500aa6..a9d3d1e 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -1,10 +1,13 @@ """FC-3a: CRUD over Source rows. FC-3c adds POST //check.""" +from pathlib import Path + from quart import Blueprint, jsonify, request from sqlalchemy import func, select from ..extensions import get_session from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source +from ..services import discord_poster_cleanup as poster_cleanup from ..services.artist_membership_service import ArtistMembershipService from ..services.artist_membership_service import rescan as membership_rescan from ..services.artist_service import ArtistService @@ -463,3 +466,49 @@ async def adopt_membership(): return jsonify({"already_tracked": exc.existing_id}) artist_id = artist.id return jsonify({"source_id": record.id, "artist_id": artist_id}), 201 + + +# -- Discord: remove posts by people outside the poster list (#4486) -------------- + +_IMAGES_ROOT = Path("/images") + + +async def _poster_cleanup_preview(source_id: int): + async with get_session() as session: + return await session.run_sync( + lambda s: poster_cleanup.preview(s, source_id=source_id) + ) + + +@sources_bp.route("//discord/other-posters", methods=["GET"]) +async def other_posters_preview(source_id: int): + """What removing other posters' posts would delete. Nothing is touched.""" + try: + projection = await _poster_cleanup_preview(source_id) + except LookupError: + return _bad("not_found", status=404) + except poster_cleanup.PosterCleanupError as exc: + return _bad("not_applicable", detail=str(exc), status=409) + projection["confirm_token"] = poster_cleanup.confirm_token(projection) + return jsonify(projection) + + +@sources_bp.route("//discord/other-posters/remove", methods=["POST"]) +async def other_posters_remove(source_id: int): + """Delete them. `confirm` must be the token of the preview the operator saw: + a poster list edited since then changes the set, and the token with it.""" + body = await request.get_json(silent=True) or {} + try: + projection = await _poster_cleanup_preview(source_id) + except LookupError: + return _bad("not_found", status=404) + except poster_cleanup.PosterCleanupError as exc: + return _bad("not_applicable", detail=str(exc), status=409) + expected = poster_cleanup.confirm_token(projection) + if body.get("confirm") != expected: + return _bad("confirm_mismatch", expected=expected) + async with get_session() as session: + result = await session.run_sync( + lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT) + ) + return jsonify(result) diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index eb340d0..c9aac02 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -39,7 +39,7 @@ class ImportSettings(Base): transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9") skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") - single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95") + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995") single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30") # Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 3501505..96901fb 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -46,6 +46,11 @@ class Source(Base): enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # alembic 0115: what the source is called on its platform, where the URL + # alone doesn't say — a Discord link is two numbers. The ingester that + # walks it refreshes it every walk, so a rename reads as the new name. + # NULL until a walk has read it, and on platforms that don't write it. + display_name: Mapped[str | None] = mapped_column(Text, nullable=True) last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) last_error: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/services/audits/single_color.py b/backend/app/services/audits/single_color.py index 6bddf07..5d955db 100644 --- a/backend/app/services/audits/single_color.py +++ b/backend/app/services/audits/single_color.py @@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since 2026-07-02 — the import-side filter (Importer._single_color_hit / SkipReason.single_color), so what the audit flags and what the import skips can never disagree. + +It is meant to catch the blank: a placeholder, an error tile, a solid fill. +Line art is the case it must not catch (#4483): a pencil doodle on white can be +well over 95% white even at full size, and a smoothing downsample blends its strokes +into the paper until it measures as blank. So the sample is taken by NEAREST +(each sampled pixel is a real pixel, strokes keep their contrast), at 256px +so thin strokes are still hit, and the default threshold is near-total +(0.995) — a few percent of ink is a drawing, not an empty image. """ from PIL import Image -_THUMB_SIZE = (64, 64) +_THUMB_SIZE = (256, 256) def evaluate( @@ -20,7 +28,8 @@ def evaluate( """True iff the fraction of pixels within `tolerance` (Euclidean RGB distance) of the dominant color exceeds `threshold`. - Downsamples to 64x64 for speed (~4ms regardless of source size). + Samples 256x256 by NEAREST (see the module docstring for why not a + smoothing resample). Alpha channels are stripped; only RGB is considered. Animated images use frame 0 (PIL's default after Image.open without seek). """ @@ -30,7 +39,7 @@ def evaluate( elif im.mode not in ("RGB", "L"): im = im.convert("RGB") if im.size != _THUMB_SIZE: - im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR) + im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST) pixels = list(im.getdata()) if not pixels: return False diff --git a/backend/app/services/discord_client.py b/backend/app/services/discord_client.py index 9d5b154..19ccf09 100644 --- a/backend/app/services/discord_client.py +++ b/backend/app/services/discord_client.py @@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content: the walk. gallery-dl skips only nested channels; one private thread the token cannot read would otherwise stop every channel after it. +And one about content: a message is taken only if it has an image or video +ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well +as post, and the operator wants the art, not the conversation (2026-09-28): a +text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is +chat. A message that passes keeps every file gallery-dl would take, numbered +as gallery-dl numbers them, so the on-disk names still match. + FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ @@ -68,6 +75,9 @@ _THREADS_BATCH = 25 # it is honoured when present; 60s is the fallback and the cap. _MAX_429_RETRIES = 4 _429_WAIT_SECONDS = 60.0 +# How far back the poster picker looks: two pages. Enough to see who posts in +# a channel, few enough to answer while the operator waits (#4491). +_POSTER_SCAN = 200 # https://discord.com/developers/docs/resources/message#message-object-message-types # DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content. @@ -79,6 +89,13 @@ _FORUM = frozenset({15, 16}) # forum, media: threads only _CATEGORY = 4 _SERVER_WALK = _TEXT | _FORUM _EMBED_TYPES = frozenset({"image", "gifv", "video"}) +# What makes a message art rather than chat: an attached file FC imports as an +# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the +# import pipeline, and this one is a network client). +_VISUAL_EXTS = frozenset({ + "png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff", + "mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv", +}) _URL_RE = re.compile( r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/" @@ -163,6 +180,82 @@ def message_text(message: dict) -> str: return "\n".join(p for p in parts if p) +def _message_and_snapshots(message: dict) -> list[dict]: + """The message itself, then each forwarded message it carries.""" + return [message] + [ + (s or {}).get("message") or {} + for s in message.get("message_snapshots") or [] + if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES + ] + + +def _is_visual(attachment: dict) -> bool: + ctype = (attachment.get("content_type") or "").lower() + if ctype.startswith(("image/", "video/")): + return True + name = attachment.get("filename") or attachment.get("url") or "" + return nameext_from_url(name)[1] in _VISUAL_EXTS + + +def has_visual_attachment(message: dict) -> bool: + """Does this message, or a message it forwards, have an image or video + attached? The one test for "art, not chat" — see the module docstring.""" + return any( + att.get("url") and _is_visual(att) + for snap in _message_and_snapshots(message) + for att in snap.get("attachments") or [] + ) + + +_SQUASH = re.compile(r"[\W_]+", re.UNICODE) +# Words a server name wraps around its creator's: "Todding's Server", +# "The Official Todding Discord". +_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"}) + + +def _squash(text: str | None) -> str: + return _SQUASH.sub("", (text or "").lower()) + + +def _server_tokens(server_name: str | None) -> list[str]: + name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ") + words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS] + tokens = [w for w in words if len(w) >= 3] + joined = "".join(words) + if len(joined) >= 3 and joined not in tokens: + tokens.append(joined) + return tokens + + +def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]: + """Mark who is probably the creator a server is named for, and order the + list for picking: suggested first, then by images posted, then messages. + + Two signals, each a reason the picker shows beside the name: + - they own the server; + - their username or display name matches the server's name, once + "'s Server", "Official", "Discord" and punctuation are taken off + ("Todding's Server" and "todding"). + Neither signal alone is proof, which is why this suggests and never + decides: the operator ticks the list. Bots are never suggested.""" + tokens = _server_tokens(server_name) + owner = str(owner_id) if owner_id else None + ranked = [] + for p in posters: + reasons = [] + if owner and p.get("id") == owner: + reasons.append("owns the server") + names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3] + if any(t in n or n in t for t in tokens for n in names): + reasons.append("name matches the server") + ranked.append({**p, "reasons": reasons, "suggested": False}) + best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0) + for r in ranked: + r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best + ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0))) + return ranked + + @dataclass class MediaItem: """One file of a Discord message. `filename`/`extension` are gallery-dl's @@ -212,6 +305,8 @@ class DiscordClient: self._server: dict = {} self._channels: dict[str, dict] = {} self._skip_feed = False + # Whose messages the walk takes (`only_from`); empty takes everyone's. + self._authors: frozenset[str] = frozenset() # -- request ----------------------------------------------------------- @@ -373,6 +468,43 @@ class DiscordClient: if meta["channel_type"] in _SERVER_WALK: yield from self._feeds(meta["channel_id"], safe=True) + def only_from(self, authors) -> None: + """Take only messages posted by these people: user ids, usernames or + display names, any case. A creator's server is full of other members + posting their own pictures; the source is subscribed to the creator. + Empty or None takes everyone's, as before.""" + self._authors = frozenset( + str(a).strip().lower() for a in authors or () if str(a).strip() + ) + + def _author_wanted(self, message: dict) -> bool: + if not self._authors: + return True + author = message.get("author") or {} + names = (author.get("id"), author.get("username"), author.get("global_name")) + return any(str(n).lower() in self._authors for n in names if n) + + def source_label(self, url: str) -> str | None: + """What the source is called in Discord — `Server · #channel`, a + thread as `#parent › thread`, a whole server by its name — from the + metadata the walk loaded. None when the walk never got that far.""" + try: + _, channel_id = parse_source_url(url) + except DiscordAPIError: + return None + server = self._server.get("server") or None + if channel_id is None: + return server + meta = self._channels.get(channel_id) or {} + name = meta.get("channel") + if not name: + return None + if name != "DMs": + name = f"#{name}" + if meta.get("is_thread") and meta.get("parent"): + name = f"#{meta['parent']} › {meta['channel']}" + return f"{server} · {name}" if server else name + def skip_feed(self) -> None: """Optional core seam (#4413): end the current channel and go on to the next one. A tick's early-out means THIS channel has nothing new, not @@ -440,6 +572,8 @@ class DiscordClient: for message in messages: if message.get("type") not in MESSAGE_TYPES: continue + if not self._author_wanted(message): + continue message["_meta"] = meta yield message, meta, page_cursor if self._skip_feed: @@ -454,13 +588,14 @@ class DiscordClient: def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]: """gallery-dl's file list for one message: attachments, then the first of video/image/thumbnail `proxy_url` of each file-bearing embed, then - the same for every forwarded snapshot; numbered from 1 across them.""" + the same for every forwarded snapshot; numbered from 1 across them. + + Empty for a message with no image or video attached: that is chat, + whatever else it carries (`has_visual_attachment`).""" + if not has_visual_attachment(post): + return [] mid = str(post.get("id") or "") - snapshots = [post] + [ - (s or {}).get("message") or {} - for s in post.get("message_snapshots") or [] - if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES - ] + snapshots = _message_and_snapshots(post) found: list[tuple[str, str, str | None]] = [] for snap in snapshots: for att in snap.get("attachments") or []: @@ -498,10 +633,10 @@ class DiscordClient: """`(message:, )` — gates the message record through the seen ledger, like `post:` on the other platforms. - None for a message with no files. gallery-dl wrote a sidecar only - beside a file, so a text-only chat line never became a post, and the - drop grouping (discord_grouping) is built on that: a channel's chatter - recorded as posts would bury the drops it exists to surface.""" + None for a message `extract_media` takes nothing from, i.e. one with + no image or video attached. The drop grouping (discord_grouping) is + built on that: a channel's chatter recorded as posts would bury the + drops it exists to surface.""" mid = post.get("id") mid = str(mid) if mid is not None else "" if not mid or not cls.extract_media(post): @@ -528,6 +663,61 @@ class DiscordClient: pass return out + def recent_posters( + self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN, + ) -> dict: + """Who has posted lately in a channel, for picking a source's poster + list (#4491): the newest `max_messages` messages, tallied per author. + Deliberately shallow — it answers "who posts here", not "who ever did". + Every poster comes back with their stable id, which is what the list + stores: a username can change on a whim, an id can't.""" + server: dict = {} + if server_id: + try: + server = self._get(f"/guilds/{server_id}") or {} + except DiscordAPIError: + server = {} + tally: dict[str, dict] = {} + before = None + seen = 0 + while seen < max_messages: + page = self._get( + f"/channels/{channel_id}/messages", + {"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before}, + ) + if not isinstance(page, list) or not page: + break + for message in page: + seen += 1 + if message.get("type") not in MESSAGE_TYPES: + continue + author = message.get("author") or {} + aid = str(author.get("id") or "") + if not aid: + continue + row = tally.setdefault(aid, { + "id": aid, + "username": author.get("username"), + "global_name": author.get("global_name"), + "bot": bool(author.get("bot")), + "messages": 0, + "images": 0, + }) + row["messages"] += 1 + if has_visual_attachment(message): + row["images"] += 1 + if len(page) < _MESSAGES_BATCH: + break + before = str(page[-1]["id"]) + return { + "server": server.get("name") or None, + "owner_id": str(server["owner_id"]) if server.get("owner_id") else None, + "scanned": seen, + "posters": rank_posters( + server.get("name"), server.get("owner_id"), list(tally.values()), + ), + } + def verify_auth(self, url: str) -> tuple[bool | None, str]: """Is the token valid, and can it see what the source names?""" try: diff --git a/backend/app/services/discord_downloader.py b/backend/app/services/discord_downloader.py index d83e354..c9f542a 100644 --- a/backend/app/services/discord_downloader.py +++ b/backend/app/services/discord_downloader.py @@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader): "parent": meta.get("parent"), "is_thread": meta.get("is_thread"), "author": author.get("username"), + "author_name": author.get("global_name"), "author_id": author.get("id"), "message": body, "date": post.get("timestamp"), diff --git a/backend/app/services/discord_ingester.py b/backend/app/services/discord_ingester.py index f83721f..0e2dff2 100644 --- a/backend/app/services/discord_ingester.py +++ b/backend/app/services/discord_ingester.py @@ -13,6 +13,11 @@ Two things differ from the cookie platforms: something; a Discord drop is routinely files and nothing else, so on Discord that is an ordinary backfill, not a broken parser. +And two things only a Discord source has: `discord_authors` in its +config_overrides limits it to the creator's own messages (a server's other +members post pictures too), and it learns its own name — the server and +channel it walks — into `source.display_name`. + `campaign_id` is the source URL (a server, channel, thread or category link). FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ @@ -20,15 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. from __future__ import annotations import asyncio +import logging from collections.abc import Callable from pathlib import Path -from ..models import DiscordFailedMedia, DiscordSeenMedia +from sqlalchemy import select, update + +from ..models import DiscordFailedMedia, DiscordSeenMedia, Source from .discord_client import DiscordAPIError, DiscordClient, MediaItem from .discord_downloader import DiscordDownloader from .ingest_core import Ingester +log = logging.getLogger(__name__) + _LEDGER_KEY_MAX = 128 +# `config_overrides` key: the people whose messages this source takes (user +# ids, usernames or display names). Absent or empty takes everyone's. +AUTHORS_KEY = "discord_authors" def _ledger_key(media: MediaItem) -> str: @@ -74,6 +87,46 @@ class DiscordIngester(Ingester): body_canary=False, ) + def run(self, **kwargs): + """The core walk, bracketed by the two things only a Discord source + has: whose messages it takes, read before the walk, and the name of + what it walks, written after it (the walk is what loads that name).""" + source_id = kwargs["source_id"] + self.client.only_from(self._source_authors(source_id)) + try: + return super().run(**kwargs) + finally: + self._record_display_name(source_id, kwargs["campaign_id"]) + + def _source_authors(self, source_id: int) -> list: + if self.session_factory is None: + return [] + with self.session_factory() as session: + overrides = session.execute( + select(Source.config_overrides).where(Source.id == source_id) + ).scalar_one_or_none() or {} + authors = overrides.get(AUTHORS_KEY) or [] + return authors if isinstance(authors, list) else [] + + def _record_display_name(self, source_id: int, url: str) -> None: + """Refreshed on every walk, never written once: a renamed channel + should read as its new name. A name the walk couldn't read leaves the + stored one alone, and a failure here never fails the walk.""" + label = self.client.source_label(url) + if not label or self.session_factory is None: + return + try: + with self.session_factory() as session: + session.execute( + update(Source) + .where(Source.id == source_id) + .where(Source.display_name.is_distinct_from(label)) + .values(display_name=label) + ) + session.commit() + except Exception as exc: # a name is decoration — never fail the walk + log.warning("Discord: couldn't record source %s's name: %s", source_id, exc) + async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]: """The uniform `(ok, message)` probe: is the token valid, and can its diff --git a/backend/app/services/discord_poster_cleanup.py b/backend/app/services/discord_poster_cleanup.py new file mode 100644 index 0000000..507f00c --- /dev/null +++ b/backend/app/services/discord_poster_cleanup.py @@ -0,0 +1,243 @@ +"""Remove a Discord source's posts by people outside its poster list (#4486). + +A creator's server is full of other members posting their own pictures. The +poster list (`discord_authors`, #4481) stops a walk taking them; this removes +the ones taken before the list existed. The operator sets the list first, then +previews what would go, then applies. + +## What goes + +A post of this source whose record names its poster (`author_id` / `author`, +written by `DiscordDownloader.write_post_record`) and names someone NOT on the +list. Two kinds of post are never touched: + +* a post whose record names no poster — a message recorded before the native + ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed; +* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled + below as a consequence, not matched as a poster's post. + +An image goes when every REAL post it belongs to is going. A link to a +synthetic drop does not keep an image: the drop only references its members' +images. An image also on a kept post — the creator re-posting a piece someone +else shared — stays. + +A drop that absorbed a removed message is deleted outright. Deleting a drop is +its undo (discord_grouping's honesty rule): its remaining members return to +the feed and the grouping sweep regroups them on its next run, so no half- +rebuilt drop keeps text or thumbnails from someone it no longer contains. + +Preview and apply spread the same predicates (rule 93, snippet #3087). +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from sqlalchemy import and_, delete, exists, func, or_, select +from sqlalchemy.orm import Session, aliased + +from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source +from .cleanup_service import delete_images +from .discord_ingester import AUTHORS_KEY + +log = logging.getLogger(__name__) + +PLATFORM = "discord" +# The record keys that name a message's poster. `author_name` (the display +# name) is only on records written after 2026-09-28. +_POSTER_KEYS = ("author_id", "author", "author_name") + + +class PosterCleanupError(ValueError): + """The source can't be cleaned this way (not Discord, or no list).""" + + +def source_authors(source: Source) -> list[str]: + """The source's poster list, lowercased; empty means everyone's wanted.""" + authors = (source.config_overrides or {}).get(AUTHORS_KEY) or [] + if not isinstance(authors, list): + return [] + return sorted({str(a).strip().lower() for a in authors if str(a).strip()}) + + +def _poster(key: str): + return func.lower(Post.raw_metadata[key].as_string()) + + +def _has_poster(): + return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author"))) + + +def _real_post_conditions(source_id: int) -> list: + return [Post.source_id == source_id, Post.synthesized_by.is_(None)] + + +def _named_poster_conditions(source_id: int) -> list: + """A real post of this source whose record says who posted it.""" + return [*_real_post_conditions(source_id), _has_poster()] + + +def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list: + """The posts that go: a named poster none of whose names is on the list.""" + return [ + *_named_poster_conditions(source_id), + *(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS), + ] + + +def _doomed_post_ids(source_id: int, authors: list[str]): + # correlate(None): used inside queries that are themselves over `post` (the + # drops, the delete), where auto-correlation would bind this to the outer + # row instead of scanning the table. + return ( + select(Post.id) + .where(*_other_poster_post_conditions(source_id, authors)) + .correlate(None) + ) + + +def _image_conditions(source_id: int, authors: list[str]) -> list: + """Images that belong to a removed post and to no kept real post.""" + doomed = _doomed_post_ids(source_id, authors) + keeper = aliased(Post) + on_doomed = or_( + ImageRecord.primary_post_id.in_(doomed), + exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.post_id.in_(doomed), + ), + ) + on_kept = or_( + and_( + ImageRecord.primary_post_id.isnot(None), + ImageRecord.primary_post_id.notin_(doomed), + ), + exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.post_id == keeper.id, + keeper.synthesized_by.is_(None), + keeper.id.notin_(doomed), + ), + ) + return [on_doomed, ~on_kept] + + +def _drop_conditions(source_id: int, authors: list[str]) -> list: + """The synthetic drops that absorbed a removed message.""" + member = aliased(Post) + return [ + Post.source_id == source_id, + Post.synthesized_by.isnot(None), + exists().where( + member.absorbed_by_post_id == Post.id, + member.id.in_(_doomed_post_ids(source_id, authors)), + ), + ] + + +def _attachment_conditions(source_id: int, authors: list[str]) -> list: + return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))] + + +def _load(session: Session, source_id: int) -> tuple[Source, list[str]]: + source = session.get(Source, source_id) + if source is None: + raise LookupError(source_id) + if source.platform != PLATFORM: + raise PosterCleanupError("Only a Discord source has posters to filter by.") + authors = source_authors(source) + if not authors: + raise PosterCleanupError( + "Set this source's 'Only posts by' list first — with no list, " + "every poster is wanted and nothing would be removed." + ) + return source, authors + + +def _count(session: Session, stmt) -> int: + return session.execute(stmt).scalar_one() + + +def preview(session: Session, *, source_id: int) -> dict: + """What `apply` would remove, per poster, without touching anything.""" + _, authors = _load(session, source_id) + who = func.coalesce( + Post.raw_metadata["author_name"].as_string(), + Post.raw_metadata["author"].as_string(), + Post.raw_metadata["author_id"].as_string(), + ) + rows = session.execute( + select(who, func.count(Post.id)) + .where(*_other_poster_post_conditions(source_id, authors)) + .group_by(who) + .order_by(func.count(Post.id).desc()) + ).all() + posters = [{"poster": name, "posts": n} for name, n in rows] + unknown = _count(session, select(func.count(Post.id)).where( + *_real_post_conditions(source_id), ~_has_poster(), + )) + return { + "authors": authors, + "posters": posters, + "posts": sum(p["posts"] for p in posters), + "images": _count(session, select(func.count(ImageRecord.id)).where( + *_image_conditions(source_id, authors))), + "attachments": _count(session, select(func.count(PostAttachment.id)).where( + *_attachment_conditions(source_id, authors))), + "drops": _count(session, select(func.count(Post.id)).where( + *_drop_conditions(source_id, authors))), + "unknown_posts": unknown, + } + + +def confirm_token(projection: dict) -> str: + """What the operator's apply must echo: the preview it was shown, so a + list edited between preview and apply can't delete an unseen set.""" + return f"remove-{projection['posts']}-posts-{projection['images']}-images" + + +def apply(session: Session, *, source_id: int, images_root: Path) -> dict: + """Remove them. Counts are taken before the deletes they describe — once + the rows are gone there is nothing left to count (snippet #3087 note 2).""" + projection = preview(session, source_id=source_id) + _, authors = _load(session, source_id) + + image_ids = session.execute( + select(ImageRecord.id).where(*_image_conditions(source_id, authors)) + ).scalars().all() + # Drops first-class: found BEFORE their members go, since the link that + # finds them is the member's absorbed_by_post_id. + drop_ids = session.execute( + select(Post.id).where(*_drop_conditions(source_id, authors)) + ).scalars().all() + + deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root) + # Attachments before posts: post_attachment.post_id is SET NULL, and a + # NULL-post attachment collides on its partial unique index (see + # cleanup_service.delete_artist_cascade). + attachments = session.execute( + delete(PostAttachment).where(*_attachment_conditions(source_id, authors)) + ).rowcount or 0 + posts = session.execute( + Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors))) + ).rowcount or 0 + drops = 0 + if drop_ids: + drops = session.execute( + Post.__table__.delete().where(Post.id.in_(drop_ids)) + ).rowcount or 0 + session.commit() + log.info( + "discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), " + "%d attachment(s), %d drop(s) removed", + source_id, authors, posts, deleted["images_deleted"], attachments, drops, + ) + return { + **projection, + "posts_deleted": posts, + "images_deleted": deleted["images_deleted"], + "files_deleted": deleted["files_deleted"], + "attachments_deleted": attachments, + "drops_deleted": drops, + } diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 979676d..167a1e0 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -13,6 +13,7 @@ from __future__ import annotations import asyncio import logging +import time from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -39,7 +40,7 @@ from .gallery_dl import ( walk_completed, ) from .importer import Importer -from .ingest_core import DEFAULT_REVISIT_DAYS +from .ingest_core import DEFAULT_REVISIT_DAYS, PHASE3_DEADLINE_SECONDS from .platforms import auth_type_for from .scheduler_service import set_platform_cooldown @@ -82,6 +83,7 @@ class DownloadService: async def download_source(self, source_id: int) -> int: """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is.""" + started = time.monotonic() setup = await self._phase1_setup(source_id) if setup["status"] in ("skipped", "in_flight"): return setup["event_id"] @@ -174,6 +176,7 @@ class DownloadService: return await self._phase3_persist( ctx["event_id"], ctx, dl_result, resolved_campaign_id, + started=started, ) async def _run_download( @@ -268,6 +271,8 @@ class DownloadService: ctx: dict, dl_result, resolved_campaign_id: str | None, + *, + started: float | None = None, ) -> int: # Cache Patreon campaign id on the source. if resolved_campaign_id: @@ -303,7 +308,23 @@ class DownloadService: loop = asyncio.get_running_loop() - for path_str in dl_result.written_paths or []: + # Phase 3 shares the task's soft limit with the walk, and an import's + # cost is only estimated there. Native platforms stop starting imports + # at the deadline: what is left stays on disk with no record, which the + # next chunk's walk imports. gallery-dl platforms have no such pickup, + # so they import everything. + deadline = ( + started + PHASE3_DEADLINE_SECONDS + if started is not None and uses_native_ingester(ctx["platform"]) + else None + ) + written_paths = list(dl_result.written_paths or []) + deferred_from: int | None = None + + for idx, path_str in enumerate(written_paths): + if deadline is not None and time.monotonic() >= deadline: + deferred_from = idx + break if path_str in (dl_result.quarantined_paths or []): continue path = Path(path_str) @@ -427,8 +448,16 @@ class DownloadService: # has been through the importer. Had the run died before here they stay # unmarked, and the next walk imports them from disk (ingest_core). mark_seen = getattr(dl_result, "mark_seen_after_import", None) + deferred = written_paths[deferred_from:] if deferred_from is not None else [] if mark_seen is not None: - await loop.run_in_executor(None, mark_seen) + if deferred: + await loop.run_in_executor( + None, lambda: mark_seen(skip_paths=frozenset(deferred)), + ) + else: + await loop.run_in_executor(None, mark_seen) + if deferred: + self._defer_unimported(dl_result, deferred_from, len(deferred)) # #830 recapture: backfill source_filehash on EXISTING on-disk images so # their post-body inline remaps to the local copy. A @@ -530,6 +559,33 @@ class DownloadService: await self.async_session.commit() return event_id + @staticmethod + def _defer_unimported(dl_result, first: int, count: int) -> None: + """Turn this chunk into a chunk boundary at the first file phase 3 left. + + The next chunk resumes on the page that file came from, so its walk + meets every left file again, on disk with no record, and imports it. A + walk that reached its end has not finished while files wait, so a clean + result becomes PARTIAL; a real failure keeps its own error. + """ + cursors = getattr(dl_result, "written_cursors", None) or [] + if first < len(cursors): + dl_result.cursor = cursors[first] + dl_result.phase3_imported_any = first > 0 + msg = ( + f"Imports stopped at the chunk deadline: {count} file(s) left " + "for the next chunk" + ) + dl_result.stdout = f"{dl_result.stdout}\n{msg}" if dl_result.stdout else msg + log.info("%s", msg) + if dl_result.error_type in (None, ErrorType.PARTIAL, ErrorType.TIMEOUT) or ( + is_informational(dl_result.error_type) + ): + dl_result.success = False + dl_result.return_code = -1 + dl_result.error_type = ErrorType.PARTIAL + dl_result.error_message = msg + async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None: """Backfill state machine (plan #693, building on the cursor of #689). @@ -597,9 +653,12 @@ class DownloadService: new_cursor = ( dl_result.cursor if uses_native_ingester(ctx["platform"]) else None ) + # Imports phase 3 finished before its deadline are progress even when + # the cursor went back to the page holding the files it left. advanced = bool( (new_cursor and new_cursor != old_cursor) or dl_result.files_downloaded > 0 + or getattr(dl_result, "phase3_imported_any", False) ) if advanced: if new_cursor: diff --git a/backend/app/services/extension_service.py b/backend/app/services/extension_service.py index 1c2168a..c4db759 100644 --- a/backend/app/services/extension_service.py +++ b/backend/app/services/extension_service.py @@ -23,6 +23,12 @@ log = logging.getLogger(__name__) # The probe runs while the chip is drawing; names that take longer are skipped. _NAME_LOOKUP_SECONDS = 6.0 +# The poster picker waits on two pages of a channel's messages (#4491). +_POSTER_LOOKUP_SECONDS = 20.0 + +# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this +# keeps the name each id was picked under, for showing the list to a person. +AUTHOR_LABELS_KEY = "discord_author_labels" class UnknownPlatformError(Exception): @@ -37,6 +43,15 @@ class UnknownArtistError(Exception): """quick-add named an `artist_id` that does not exist.""" +class PosterLookupError(Exception): + """The Discord poster picker couldn't read the channel (no token, no + access, not a channel page, or Discord was slow).""" + + +class UnknownSourceError(Exception): + """No source follows this Discord channel or its server.""" + + # Mirrored byte-for-byte from extension/lib/platforms.js # PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand — # reviewers catch drift. @@ -110,6 +125,7 @@ class ExtensionService: artist_id: int | None = None, artist_name: str | None = None, use_platform_name: bool = False, + discord_authors: list[dict] | None = None, ) -> dict: """Add `url` as a source. `artist_id` connects it to an existing artist, `artist_name` to that artist (created if new); with neither, @@ -119,7 +135,10 @@ class ExtensionService: name is canon: a Patreon source added to an existing artist renames that artist to the creator's Patreon display name. Name only — the slug, and every path keyed off it, never moves (#130). Ignored on every - other platform, and when the name can't be read.""" + other platform, and when the name can't be read. + + `discord_authors` ([{id, name}], #4491) sets a Discord source's poster + list in the same step, on a new source or one already there.""" platform, raw_slug = self._derive(url) url = canonical_source_url(platform, url, raw_slug) renamed_from = None @@ -133,6 +152,8 @@ class ExtensionService: artist = (await self.session.execute( select(Artist).where(Artist.id == existing.artist_id) )).scalar_one() + if platform == DISCORD and discord_authors is not None: + await self._apply_posters(existing, discord_authors) return self._shape(existing, artist, created_source=False, created_artist=False) if artist_id is not None: @@ -154,11 +175,98 @@ class ExtensionService: source, created_source = await self._find_or_create_source( artist_id=artist.id, platform=platform, url=url, ) + if platform == DISCORD and discord_authors is not None: + await self._apply_posters(source, discord_authors) shaped = self._shape(source, artist, created_source, created_artist) if renamed_from is not None: shaped["renamed_from"] = renamed_from return shaped + # -- Discord poster picker (#4491) ------------------------------------------ + + def _discord_page(self, url: str) -> tuple[str, str]: + platform, raw_slug = self._derive(url) + if platform != DISCORD: + raise InvalidUrlError("not a Discord page") + server_id, _, channel_id = raw_slug.partition("/") + if not channel_id: + raise PosterLookupError("Open a channel to see who posts in it.") + return server_id, channel_id + + async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None: + """The source this channel's posts land on: the channel's own, else a + whole-server source that walks it.""" + base = f"https://discord.com/channels/{server_id}" + return ( + await self._existing_source(DISCORD, f"{base}/{channel_id}") + or await self._existing_source(DISCORD, base) + ) + + async def discord_posters(self, url: str) -> dict: + """Who posts in the channel `url` shows, newest `_POSTER_SCAN` + messages deep, with the likely creator marked — and, when a source + already follows it, the poster list that source has now.""" + server_id, channel_id = self._discord_page(url) + import asyncio + + from .credential_service import CredentialService + from .discord_client import DiscordAPIError, DiscordClient + token = None + if self._crypto is not None: + token = await CredentialService(self.session, self._crypto).get_token(DISCORD) + if not token: + raise PosterLookupError("No Discord token is saved in FabledCurator.") + client = DiscordClient(token, max_retries=0) + loop = asyncio.get_running_loop() + try: + found = await asyncio.wait_for( + loop.run_in_executor(None, client.recent_posters, server_id, channel_id), + timeout=_POSTER_LOOKUP_SECONDS, + ) + except TimeoutError as exc: + raise PosterLookupError("Discord took too long to answer; try again.") from exc + except DiscordAPIError as exc: + raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc + source = await self._source_for_page(server_id, channel_id) + co = (source.config_overrides or {}) if source is not None else {} + from .discord_ingester import AUTHORS_KEY + return { + **found, + "source_id": source.id if source is not None else None, + "selected": list(co.get(AUTHORS_KEY) or []), + # So someone on the list who hasn't posted lately still shows by + # name, and can be unticked. + "selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}), + } + + async def set_discord_posters(self, url: str, authors: list[dict]) -> dict: + """Store the picked posters on the source that follows `url`'s channel.""" + server_id, channel_id = self._discord_page(url) + source = await self._source_for_page(server_id, channel_id) + if source is None: + raise UnknownSourceError("No source follows this channel yet.") + return await self._apply_posters(source, authors) + + async def _apply_posters(self, source: Source, authors: list[dict]) -> dict: + """Ids are what the list keys on — a username can change on a whim, + an id can't — and the name each was picked under rides beside it for + showing the list to a person. An empty pick clears the list, which + takes everyone's posts again.""" + from .discord_ingester import AUTHORS_KEY + picked = [a for a in authors if str(a.get("id") or "").strip()] + co = dict(source.config_overrides or {}) + if picked: + co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked] + co[AUTHOR_LABELS_KEY] = { + str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked + } + else: + co.pop(AUTHORS_KEY, None) + co.pop(AUTHOR_LABELS_KEY, None) + source.config_overrides = co or None + await self.session.commit() + return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])} + async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None: """Rename `artist` to the Patreon display name; the old name when it changed, else None. Unreadable name → no rename, never the handle.""" diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 4ee68ae..98ce034 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -169,6 +169,10 @@ class DownloadResult: # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty # on the gallery-dl path and outside recapture. relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list) + # Native ingester only: one entry per `written_paths` entry, the backfill + # cursor of the page that file came from. Phase 3 resumes the next chunk + # there when it leaves files unimported at its deadline. Empty on gallery-dl. + written_cursors: list[str | None] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 @@ -194,8 +198,10 @@ class DownloadResult: # but not yet imported is invisible to every later walk, so a run killed in # between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13 # images to a stranded run, 2026-09-24). Unmarked, the next walk finds the - # file on disk with no ImageRecord and imports it. None on gallery-dl. - mark_seen_after_import: Callable[[], None] | None = None + # file on disk with no ImageRecord and imports it. Takes `skip_paths`: the + # written paths phase 3 did not import, which stay unmarked for the same + # reason. None on gallery-dl. + mark_seen_after_import: Callable[..., None] | None = None def extract_errors_warnings(stderr: str) -> str: diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 2c8a859..2ee9154 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -114,13 +114,29 @@ _CANARY_MIN_SAMPLE = 30 # on-disk files, and died at the soft limit. # # So the walk also stops when its elapsed time PLUS phase 3's estimated cost -# would pass CHUNK_TOTAL_SECONDS. Costs measured on the live instance: 431 -# imports took 976s (~2.3s each: hash, pHash, sidecar, provenance); a relink is -# a sha256 over NFS, 0.15s for the 8.8 MB average file, plus a lookup. -# test_download_source_task pins CHUNK_TOTAL_SECONDS under the soft limit. +# would pass CHUNK_TOTAL_SECONDS. An import (hash, pHash, sidecar, provenance, +# thumbnail enqueue) runs 2-5s depending on the files; the estimate takes the +# slow end. A relink is a sha256 over NFS plus a lookup. +# +# The estimate is a plan, not a guard. PHASE3_DEADLINE_SECONDS is the guard: +# phase 3 starts no import past it (download_service), and leaves the rest on +# disk, unmarked, for the next chunk. It sits under the soft limit by enough to +# finish the import in hand, upsert post records and finalize the event. +# test_download_source_task pins both under the soft limit. CHUNK_TOTAL_SECONDS = 1200.0 -PHASE3_IMPORT_SECONDS = 2.5 +PHASE3_IMPORT_SECONDS = 5.0 PHASE3_RELINK_SECONDS = 0.25 +PHASE3_DEADLINE_SECONDS = 1200.0 + + +def _unskipped( + fetched: list[tuple[str, str]], + written_keys: dict[str, tuple[str, str]], + skip_paths, +) -> list[tuple[str, str]]: + """`fetched` without the ledger keys of files phase 3 left unimported.""" + skip = {written_keys[p] for p in skip_paths if p in written_keys} + return [kv for kv in fetched if kv not in skip] if skip else list(fetched) def _parse_published(raw: object) -> datetime | None: @@ -289,6 +305,12 @@ class Ingester: # Media handed to phase 3 for import. Marked seen by phase 3 once the # import has run (`mark_seen_after_import`), not here — see there. fetched: list[tuple[str, str]] = [] + # Per entry of `written`: the cursor of the page it came from, and its + # ledger key. Phase 3 can stop importing at its deadline; the files it + # leaves must stay unmarked, and the next chunk has to resume on the + # page holding the first of them, or no walk sees them again. + written_cursors: list[str | None] = [] + written_keys: dict[str, tuple[str, str]] = {} # Post-record keys written this walk. Marked with the media, after # phase 3 has upserted the records — marking them at write time left a # walk killed before phase 3 with posts the ledger calls recorded that @@ -347,8 +369,10 @@ class Ingester: written_paths=written, post_record_paths=list(post_records), relink_source_paths=list(relink), - mark_seen_after_import=lambda: self._mark_seen( - source_id, fetched + recorded, + written_cursors=list(written_cursors), + mark_seen_after_import=lambda skip_paths=(): self._mark_seen( + source_id, + _unskipped(fetched, written_keys, skip_paths) + recorded, ), stdout="\n".join(log_lines), stderr="", @@ -577,6 +601,8 @@ class Ingester: downloaded += 1 if outcome.path is not None: written.append(str(outcome.path)) + written_cursors.append(emitted_cursor) + written_keys[str(outcome.path)] = (key, media_item.post_id) fetched.append((key, media_item.post_id)) to_clear.append(key) consecutive_seen = 0 @@ -590,6 +616,8 @@ class Ingester: # attach_in_place because no record owns this path — # the unlink below is about a file that IS the record. written.append(str(outcome.path)) + written_cursors.append(emitted_cursor) + written_keys[str(outcome.path)] = (key, media_item.post_id) fetched.append((key, media_item.post_id)) to_clear.append(key) skipped_count += 1 diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index e41b5d2..36deb48 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -479,6 +479,10 @@ class PostFeedService: # the UI uses this to explain why a post it linked to is not in the # stream. "absorbed_by_post_id": post.absorbed_by_post_id, + # #4481. The Discord channel a message was posted in: both the native + # post record and gallery-dl's sidecars carry it as `channel`. None + # on every other platform, and on a Discord post that never said. + "channel": _discord_channel(post, source), "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, "source": ( {"id": source.id, "platform": source.platform} @@ -488,3 +492,13 @@ class PostFeedService: "thumbnails_more": thumbs_entry["more"], "attachments": atts_map.get(post.id, []), } + + +def _discord_channel(post: Post, source: Source | None) -> str | None: + if source is None or source.platform != "discord": + return None + raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {} + channel = raw.get("channel") + if not isinstance(channel, str): + return None + return channel.strip() or None diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 42201a7..4b5130a 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -68,6 +68,9 @@ class SourceRecord: artist_slug: str platform: str url: str + # alembic 0115: the name the platform gives it, where the URL is opaque + # (a Discord link is two ids). None until a walk has read it. + display_name: str | None enabled: bool config_overrides: dict | None last_checked_at: str | None @@ -110,6 +113,7 @@ class SourceRecord: "artist_slug": self.artist_slug, "platform": self.platform, "url": self.url, + "display_name": self.display_name, "enabled": self.enabled, "config_overrides": self.config_overrides, "last_checked_at": self.last_checked_at, @@ -301,6 +305,7 @@ class SourceService: artist_slug=artist.slug, platform=source.platform, url=source.url, + display_name=source.display_name, enabled=source.enabled, config_overrides=source.config_overrides, last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None, diff --git a/extension/background/background.js b/extension/background/background.js index f41d117..74b62d0 100644 --- a/extension/background/background.js +++ b/extension/background/background.js @@ -276,11 +276,26 @@ browser.runtime.onMessage.addListener(async (msg) => { artistId: msg.artistId ?? null, artistName: msg.artistName ?? null, usePlatformName: msg.usePlatformName === true, + discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null, }); } catch (e) { return { error: e.message }; } + case 'DISCORD_POSTERS': + try { + return await api.getDiscordPosters(msg.url); + } catch (e) { + return { error: e.message }; + } + + case 'SET_DISCORD_POSTERS': + try { + return await api.setDiscordPosters(msg.url, msg.authors || []); + } catch (e) { + return { error: e.message }; + } + case 'SEARCH_ARTISTS': try { return { artists: await api.searchArtists(msg.q || '') }; diff --git a/extension/content/content-script.css b/extension/content/content-script.css index 86eaa64..7043f29 100644 --- a/extension/content/content-script.css +++ b/extension/content/content-script.css @@ -100,3 +100,7 @@ /* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */ .fc-panel [hidden] { display: none !important; } .fc-panel__rename { margin-top: 8px; font-size: 13px; } +/* Poster picker (#4491): who posts in the channel, the likely creator ticked. */ +.fc-panel__posters { max-height: 220px; overflow-y: auto; } +.fc-panel__poster { flex-wrap: wrap; } +.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); } diff --git a/extension/content/content-script.js b/extension/content/content-script.js index 6f3bacf..8b3380e 100644 --- a/extension/content/content-script.js +++ b/extension/content/content-script.js @@ -81,6 +81,13 @@ const probe = currentProbe; if (probe?.state === 'source_match') { + // A followed Discord channel opens its poster list (#4491); the artist + // page is one click further, in that panel. + if (probe.platform === 'discord' && probe.discord?.channel_id) { + if (document.getElementById('fc-add-panel')) closePanel(); + else openPostersPanel(btn, probe); + return; + } await openArtist(btn, probe.artist?.slug); return; } @@ -158,6 +165,120 @@ document.getElementById('fc-add-panel')?.remove(); } + // ---- Poster picker (#4491) ---- + // Who posted in this channel lately, read by FabledCurator with its Discord + // token, the creator the server is named for (or owned by) ticked. The + // source keeps their ids: a username can change on a whim, an id can't, so + // the list never quietly stops matching. Names are other people's text — + // createElement only. + + function posterSection() { + const list = el('div', { class: 'fc-panel__posters' }, [ + el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }), + ]); + const node = el('div', {}, [ + el('div', { class: 'fc-panel__label', text: 'Only posts by' }), + list, + ]); + const state = { data: null, picked: new Set() }; + + function render() { + const posters = withSelected(state.data); + if (!posters.length) { + list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' })); + return; + } + const rows = posters.map((p) => { + const id = String(p.id); + const box = el('input', { type: 'checkbox', checked: state.picked.has(id) }); + box.addEventListener('change', () => { + if (box.checked) state.picked.add(id); + else state.picked.delete(id); + hint.textContent = pickHint(); + }); + const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`]; + if (p.reasons?.length) bits.unshift(p.reasons.join(', ')); + return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [ + box, + el('span', { text: posterLabel(p) }), + el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }), + ]); + }); + const hint = el('div', { class: 'fc-panel__hint', text: pickHint() }); + list.replaceChildren(...rows, hint); + } + + function pickHint() { + return state.picked.size + ? 'Only the ticked people’s posts are taken.' + : 'Nothing ticked: everyone’s posts are taken.'; + } + + async function load() { + let r; + try { + r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href }); + } catch (e) { + r = { error: e.message }; + } + if (r?.error) { + list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` })); + return; + } + state.data = r; + state.picked = initialPosterPick(r); + render(); + } + + // null until the list has loaded: an add made before then leaves the + // source's list as it is instead of clearing it. + const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null); + return { node, load, request }; + } + + function openPostersPanel(btn, probe) { + closePanel(); + const d = probe.discord || {}; + const posters = posterSection(); + const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' }); + const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` }); + const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' }); + const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [ + el('div', { class: 'fc-panel__title', text: 'Discord source' }), + el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }), + posters.node, + el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]), + ]); + panel.addEventListener('keydown', (e) => { + e.stopPropagation(); + if (e.key === 'Escape') closePanel(); + }); + cancelBtn.addEventListener('click', closePanel); + artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); }); + saveBtn.addEventListener('click', async () => { + const authors = posters.request(); + if (!authors) return; + saveBtn.disabled = true; + try { + const r = await browser.runtime.sendMessage({ + type: 'SET_DISCORD_POSTERS', url: window.location.href, authors, + }); + if (r?.error) { + showToast(`Error: ${r.error}`, 'error'); + return; + } + showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success'); + closePanel(); + } catch (e) { + showToast(`Error: ${e.message}`, 'error'); + } finally { + saveBtn.disabled = false; + } + }); + document.body.appendChild(panel); + posters.load(); + } + async function openAddPanel(btn, probe) { closePanel(); const platformName = PLATFORMS[probe.platform]?.name || probe.platform; @@ -184,6 +305,8 @@ const d = probe.discord || {}; const discord = probe.platform === 'discord'; const choice = panelDefaults(probe, window.location.href); + // #4491: on a channel page, pick whose posts the new source takes. + const posters = discord && d.channel_id ? posterSection() : null; const scopeRow = (value, label, disabled) => { const input = el('input', { @@ -234,6 +357,7 @@ el('div', { class: 'fc-panel__label', text: 'Artist' }), el('div', { class: 'fc-panel__combo' }, [nameInput, results]), renameRow, + ...(posters ? [posters.node] : []), hint, el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]), ]); @@ -392,6 +516,8 @@ addBtn.addEventListener('click', async () => { const req = addRequest(choice); if (!req) return; + const authors = posters?.request(); + if (authors) req.discordAuthors = authors; const btn = document.getElementById('fc-add-source-btn'); addBtn.disabled = true; const ok = await add(btn || addBtn, req); @@ -402,6 +528,7 @@ document.body.appendChild(panel); refresh(); nameInput.focus(); + posters?.load(); nameInput.select(); // Search what the field opens with — the server's name, usually — so an // artist it already matches is picked before the operator types anything. diff --git a/extension/lib/api.js b/extension/lib/api.js index aa9fe0b..de26d28 100644 --- a/extension/lib/api.js +++ b/extension/lib/api.js @@ -106,8 +106,11 @@ class FabledCuratorAPI { // artist from the URL. A Discord channel always sends one. // usePlatformName: a Patreon source joining an existing artist renames it // to the Patreon display name (Patreon is canon; name only, never the slug). - quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false } = {}) { + // discordAuthors ([{id, name}], #4491): a Discord source's poster list, set + // in the same step. + quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) { const body = { url }; + if (discordAuthors) body.discord_authors = discordAuthors; if (artistId != null) body.artist_id = artistId; else if (artistName) body.artist_name = artistName; if (usePlatformName) body.use_platform_name = true; @@ -123,6 +126,15 @@ class FabledCuratorAPI { const qs = new URLSearchParams(params).toString(); return this.request('GET', `/extension/probe?${qs}`); } + // #4491: who posts in the Discord channel `url` shows, lately, with the + // likely creator marked; and setting the list on the source that follows it. + getDiscordPosters(url) { + const qs = new URLSearchParams({ url }).toString(); + return this.request('GET', `/extension/discord/posters?${qs}`); + } + setDiscordPosters(url, authors) { + return this.request('POST', '/extension/discord/posters', { url, authors }); + } // Latest published extension version on this instance — drives the in-app // update prompt. Public endpoint (no key needed, but request() sends it // harmlessly). Returns {version, xpi_url, latest_url, sha256}. diff --git a/extension/lib/chip.js b/extension/lib/chip.js index 1a7f0f3..76af646 100644 --- a/extension/lib/chip.js +++ b/extension/lib/chip.js @@ -144,3 +144,49 @@ function inlineCompletion(typed, results) { if (!t) return null; return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null; } + +// ---- Discord poster picker (#4491) ---- +// /api/extension/discord/posters lists who posted in the channel lately, with +// the creator the server is named for (or owned by) marked `suggested`. The +// picker ticks posters; the source keeps their ids, since a username can +// change on a whim and an id can't. + +/** "Todding (@todding)" — the display name, then the handle when it differs. */ +function posterLabel(p) { + const shown = p?.global_name || p?.username || p?.id || ''; + const handle = p?.username && p.username !== shown ? ` (@${p.username})` : ''; + return `${shown}${handle}`; +} + +/** What the picker starts with: the source's current list when it has one, + * else the suggested creator(s). A set of ids. */ +function initialPosterPick(data) { + const current = (data?.selected || []).map(String); + if (current.length) return new Set(current); + return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id))); +} + +/** The checklist's rows: who posted lately, then anyone already on the + * source's list who hasn't — by the name they were picked under — so they + * can still be unticked. */ +function withSelected(data) { + const posters = [...(data?.posters || [])]; + const shown = new Set(posters.map((p) => String(p.id))); + const labels = data?.selected_labels || {}; + for (const id of (data?.selected || []).map(String)) { + if (shown.has(id)) continue; + posters.push({ id, global_name: labels[id] || null, username: null, + messages: 0, images: 0, reasons: ['on the list, not posting lately'] }); + } + return posters; +} + +/** The picked posters as the API takes them: [{id, name}], in list order. + * An id already on the source but no longer posting lately is kept. */ +function postersRequest(posters, picked) { + const byId = new Map((posters || []).map((p) => [String(p.id), p])); + return [...picked].map((id) => ({ + id, + name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null, + })); +} diff --git a/extension/test/chip.spec.js b/extension/test/chip.spec.js index 2edfa43..d887c1f 100644 --- a/extension/test/chip.spec.js +++ b/extension/test/chip.spec.js @@ -198,3 +198,46 @@ describe('Add panel on Patreon and SubscribeStar', () => { expect(c.artistName).toBe('Tamada') }) }) + +const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [ + 'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected', +]) + +describe('the poster checklist rows', () => { + it('adds someone on the list who has not posted lately, by their saved name', () => { + const rows = withSelected({ + posters: [{ id: '7', username: 'todding', global_name: 'Todding' }], + selected: ['7', '42'], + selected_labels: { 42: 'Old Name' }, + }) + expect(rows.map((r) => r.id)).toEqual(['7', '42']) + expect(posterLabel(rows[1])).toBe('Old Name') + expect(rows[1].reasons).toEqual(['on the list, not posting lately']) + }) +}) + +describe('the Discord poster picker (#4491)', () => { + const posters = [ + { id: '7', username: 'todding', global_name: 'Todding', suggested: true }, + { id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false }, + { id: '9', username: 'plain', global_name: null, suggested: false }, + ] + + it('labels a poster by display name, then handle', () => { + expect(posterLabel(posters[0])).toBe('Todding (@todding)') + expect(posterLabel(posters[2])).toBe('plain') + }) + + it('starts from the source list when it has one, else the suggestion', () => { + expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7']) + expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8']) + expect([...initialPosterPick({ posters: [] })]).toEqual([]) + }) + + it('sends ids with the names they were picked under, keeping unknown ids', () => { + expect(postersRequest(posters, new Set(['7', '42']))).toEqual([ + { id: '7', name: 'Todding' }, + { id: '42', name: null }, + ]) + }) +}) diff --git a/frontend/src/components/cleanup/SingleColorAuditCard.vue b/frontend/src/components/cleanup/SingleColorAuditCard.vue index 1a87772..5b16302 100644 --- a/frontend/src/components/cleanup/SingleColorAuditCard.vue +++ b/frontend/src/components/cleanup/SingleColorAuditCard.vue @@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue' import { useCleanupStore } from '../../stores/cleanup.js' const store = useCleanupStore() -const threshold = ref(0.95) +const threshold = ref(0.995) const tolerance = ref(30) const audit = ref(null) const busy = ref(false) diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index d8a7981..07a22ef 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -26,6 +26,7 @@ :to="{ name: 'artist', params: { slug: post.artist.slug } }" class="fc-post-card__artist" >{{ post.artist.name }} + #{{ post.channel }} {{ relativeDate }} · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }} diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue index 176def1..84a31e0 100644 --- a/frontend/src/components/settings/ImportFiltersForm.vue +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -77,7 +77,7 @@ @@ -142,7 +142,7 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' } const local = reactive({ min_width: 0, min_height: 0, skip_transparent: false, transparency_threshold: 0.9, - skip_single_color: false, single_color_threshold: 0.95, + skip_single_color: false, single_color_threshold: 0.995, phash_threshold: 24, wip_title_tagging_enabled: true, }) diff --git a/frontend/src/components/subscriptions/PosterCleanupDialog.vue b/frontend/src/components/subscriptions/PosterCleanupDialog.vue new file mode 100644 index 0000000..cc2873b --- /dev/null +++ b/frontend/src/components/subscriptions/PosterCleanupDialog.vue @@ -0,0 +1,135 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 3512b70..03e4ddc 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -50,6 +50,17 @@ library — without re-downloading media + + Remove posts from other posters + + Preview, then delete the posts taken from people outside this + source's "Only posts by" list + + Remove source +