diff --git a/Dockerfile b/Dockerfile index 8f4d36e..17d7944 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,13 +18,16 @@ ENV PYTHONUNBUFFERED=1 \ # System deps: ffmpeg (transcode + thumbnails, FC-2), unar (archives, FC-2), # libpq for psycopg, postgresql-client + zstd for FC-5 backup/restore -# (pg_dump + tar --zstd), image libs. +# (pg_dump + tar --zstd), image libs, megatools (mega.nz public-link downloads +# for off-platform file-host links, #830 — `megatools dl`; Debian-native, no +# external MEGA apt repo needed). RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ unar \ libpq5 \ postgresql-client \ zstd \ + megatools \ libjpeg62-turbo \ libwebp7 \ libpng16-16 \ diff --git a/alembic/versions/0049_external_link_table.py b/alembic/versions/0049_external_link_table.py new file mode 100644 index 0000000..373c807 --- /dev/null +++ b/alembic/versions/0049_external_link_table.py @@ -0,0 +1,90 @@ +"""external_link table — off-platform file-host links found in post bodies + +Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox / +Pixeldrain and link them in the post text. This table records each such link +(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger +the download worker (a later slice) walks. `url` keeps the FULL link including +the `#fragment` — mega.nz's decryption key lives there; truncating it makes the +file undownloadable. + +CHECK whitelists for host + status include the full enum up front (incl. the +download-worker statuses) so the worker slice needs no constraint migration. + +Revision ID: 0049 +Revises: 0048 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0049" +down_revision: Union[str, None] = "0048" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "external_link", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "post_id", sa.Integer(), + sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column( + "artist_id", sa.Integer(), + sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("host", sa.String(length=16), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="pending", + ), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "attachment_id", sa.Integer(), + sa.ForeignKey("post_attachment.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("duration_seconds", sa.Float(), nullable=True), + sa.CheckConstraint( + "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')", + name="ck_external_link_host", + ), + sa.CheckConstraint( + "status IN ('pending','downloading','downloaded','failed'," + "'skipped','dead')", + name="ck_external_link_status", + ), + ) + op.create_index( + "ix_external_link_post_id", "external_link", ["post_id"], + ) + op.create_index( + "ix_external_link_artist_id", "external_link", ["artist_id"], + ) + op.create_index( + "ix_external_link_status", "external_link", ["status"], + ) + op.create_index( + "uq_external_link_post_url", "external_link", ["post_id", "url"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("uq_external_link_post_url", table_name="external_link") + op.drop_index("ix_external_link_status", table_name="external_link") + op.drop_index("ix_external_link_artist_id", table_name="external_link") + op.drop_index("ix_external_link_post_id", table_name="external_link") + op.drop_table("external_link") diff --git a/alembic/versions/0050_external_link_host_toggles.py b/alembic/versions/0050_external_link_host_toggles.py new file mode 100644 index 0000000..ac78e75 --- /dev/null +++ b/alembic/versions/0050_external_link_host_toggles.py @@ -0,0 +1,38 @@ +"""import_settings: per-host enable toggles for external file-host downloads + +Operator levers (#830): disable a single host (e.g. mega.nz when it's +rate-limiting/banning) without touching the others. The worker reads these via +getattr and defaults to enabled, so the toggles default TRUE (works out of the +box, rule #26). + +Revision ID: 0050 +Revises: 0049 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0050" +down_revision: Union[str, None] = "0049" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") + + +def upgrade() -> None: + for host in _HOSTS: + op.add_column( + "import_settings", + sa.Column( + f"extdl_{host}_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + for host in _HOSTS: + op.drop_column("import_settings", f"extdl_{host}_enabled") diff --git a/alembic/versions/0051_image_source_provenance.py b/alembic/versions/0051_image_source_provenance.py new file mode 100644 index 0000000..595077d --- /dev/null +++ b/alembic/versions/0051_image_source_provenance.py @@ -0,0 +1,38 @@ +"""image_record: source_url + source_filehash (inline-image localization) + +#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline +images instead of hotlinking the public CDN. The join key between a body +`` and the local file is the CDN's 32-hex filehash (the same +identity extract_media dedups by). Persist it (indexed) plus the full source +URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing +rows — those fall back to hotlinking until re-downloaded. + +Revision ID: 0051 +Revises: 0050 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0051" +down_revision: Union[str, None] = "0050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True)) + op.add_column( + "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True) + ) + op.create_index( + "ix_image_record_source_filehash", "image_record", ["source_filehash"] + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_source_filehash", table_name="image_record") + op.drop_column("image_record", "source_filehash") + op.drop_column("image_record", "source_url") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 7d69fbf..a8af431 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -27,6 +27,20 @@ _EDITABLE_FIELDS = ( "download_failure_warning_threshold", "series_suggest_enabled", "series_suggest_threshold", + "extdl_mega_enabled", + "extdl_gdrive_enabled", + "extdl_mediafire_enabled", + "extdl_dropbox_enabled", + "extdl_pixeldrain_enabled", +) + +# Per-host external-download toggles — all plain booleans, validated uniformly. +_EXTDL_TOGGLE_FIELDS = ( + "extdl_mega_enabled", + "extdl_gdrive_enabled", + "extdl_mediafire_enabled", + "extdl_dropbox_enabled", + "extdl_pixeldrain_enabled", ) @@ -50,6 +64,11 @@ async def get_import_settings(): "download_failure_warning_threshold": row.download_failure_warning_threshold, "series_suggest_enabled": row.series_suggest_enabled, "series_suggest_threshold": row.series_suggest_threshold, + "extdl_mega_enabled": row.extdl_mega_enabled, + "extdl_gdrive_enabled": row.extdl_gdrive_enabled, + "extdl_mediafire_enabled": row.extdl_mediafire_enabled, + "extdl_dropbox_enabled": row.extdl_dropbox_enabled, + "extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled, }) @@ -106,6 +125,9 @@ async def update_import_settings(): return jsonify( {"error": "series_suggest_enabled must be a boolean"} ), 400 + for tog in _EXTDL_TOGGLE_FIELDS: + if tog in body and not isinstance(body[tog], bool): + return jsonify({"error": f"{tog} must be a boolean"}), 400 if "series_suggest_threshold" in body: v = body["series_suggest_threshold"] if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 2d39f8f..0576d85 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -30,6 +30,7 @@ def make_celery() -> Celery: "backend.app.tasks.maintenance", "backend.app.tasks.ml", "backend.app.tasks.download", + "backend.app.tasks.external", "backend.app.tasks.backup", "backend.app.tasks.admin", "backend.app.tasks.library_audit", @@ -42,6 +43,9 @@ def make_celery() -> Celery: "backend.app.tasks.ml.*": {"queue": "ml"}, "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"}, "backend.app.tasks.download.*": {"queue": "download"}, + # External file-host fetches are downloads — same lane (they can run + # long, but the download worker already tolerates long backfills). + "backend.app.tasks.external.*": {"queue": "download"}, "backend.app.tasks.scan.*": {"queue": "scan"}, # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup # (concurrency-1 on the scheduler). The long one-shots (DB backups, @@ -150,6 +154,21 @@ def make_celery() -> Celery: "task": "backend.app.tasks.thumbnail.backfill_thumbnails", "schedule": 86400.0, }, + # External file-host downloads (#830): a steady sweep catches links + # the post-download hook missed (worker down, etc.); recovery re-tries + # dead links daily; retention prunes long-dead rows. + "extdl-sweep": { + "task": "backend.app.tasks.external.sweep_external_links", + "schedule": 600.0, # every 10 min + }, + "extdl-recover-daily": { + "task": "backend.app.tasks.external.recover_external_links", + "schedule": 86400.0, + }, + "extdl-prune-daily": { + "task": "backend.app.tasks.external.prune_external_links", + "schedule": 86400.0, + }, }, timezone="UTC", ) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 19d444b..4433dc2 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -7,6 +7,7 @@ from .backup_run import BackupRun from .base import Base from .credential import Credential from .download_event import DownloadEvent +from .external_link import ExternalLink from .image_prediction import ImagePrediction from .image_provenance import ImageProvenance from .image_record import ImageRecord @@ -52,6 +53,7 @@ __all__ = [ "TagKind", "image_tag", "DownloadEvent", + "ExternalLink", "ImportBatch", "ImportTask", "ImportSettings", diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py new file mode 100644 index 0000000..0902e28 --- /dev/null +++ b/backend/app/models/external_link.py @@ -0,0 +1,73 @@ +"""ExternalLink — an off-platform file-host link found in a post body. + +Creators host the actual files (films, packs) on mega.nz / Google Drive / +MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row +is the record that the link existed (so nothing is silently dropped), the +dedup + dead-letter ledger for fetching it, and the driver the download worker +walks. `url` keeps the FULL link including the `#fragment` (mega's decryption +key) — truncating it makes the file undownloadable. + +status lifecycle: pending → downloading → downloaded | failed | dead +(too many attempts) | skipped (host disabled). `attachment_id` links the +captured file once a download lands (SET NULL so deleting the attachment +doesn't delete the link record). +""" + +from datetime import datetime + +from sqlalchemy import ( + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + func, + text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + +# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049. +HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") +STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead") + + +class ExternalLink(Base): + __tablename__ = "external_link" + __table_args__ = ( + # One row per (post, url). The full url (incl. #fragment) is the identity + # — the same file linked twice in a post collapses to one row. + Index("uq_external_link_post_url", "post_id", "url", unique=True), + Index("ix_external_link_status", "status"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + artist_id: Mapped[int | None] = mapped_column( + ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True + ) + host: Mapped[str] = mapped_column(String(16), nullable=False) + url: Mapped[str] = mapped_column(Text, nullable=False) + label: Mapped[str | None] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending" + ) + attempts: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + attachment_id: Mapped[int | None] = mapped_column( + ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index ac57c9a..07cb0ec 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -49,6 +49,18 @@ class ImageRecord(Base): # Thumbnail (populated by FC-2) thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True) + # Source provenance for downloaded media (#830 Phase 2). `source_url` is the + # CDN/origin URL the file was fetched from (debugging + future re-fetch). + # `source_filehash` is the URL's 32-hex CDN identity segment + # (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's + # inline `` back to this local copy so the rendered body serves + # our stored image instead of hotlinking the public source. Indexed for the + # render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows. + source_url: Mapped[str | None] = mapped_column(Text, nullable=True) + source_filehash: Mapped[str | None] = mapped_column( + String(32), nullable=True, index=True + ) + # Origin / provenance pointers origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False) primary_post_id: Mapped[int | None] = mapped_column( diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index 8c59fa8..5b533fc 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -73,6 +73,25 @@ class ImportSettings(Base): Float, nullable=False, default=0.5, ) + # #830 off-platform file-host downloads — per-host enable lever (default on, + # rule #26). Column names are extdl__enabled so the worker reads them + # via getattr(settings, f"extdl_{host}_enabled", True). + extdl_mega_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="true", + ) + extdl_gdrive_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="true", + ) + extdl_mediafire_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="true", + ) + extdl_dropbox_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="true", + ) + extdl_pixeldrain_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="true", + ) + @classmethod async def load(cls, session) -> ImportSettings: """The singleton settings row (id=1), via an async session.""" diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 0fc04e6..479013f 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -380,6 +380,29 @@ class DownloadService: else: import_summary["errors"] += 1 + # Post-only records: media-less posts (pure-text) the native ingester + # captured so the artist archive is complete. Upsert each (keyed on + # external_post_id → updates the same Post a media import would create, + # never doubles). No file to clean up; the sidecar stays on disk. + for rec_str in getattr(dl_result, "post_record_paths", None) or []: + rec_path = Path(rec_str) + if not rec_path.exists(): # noqa: ASYNC240 + continue + + def _upsert(p=rec_path): + return self.importer.upsert_post_record( + p, artist=artist, source=source_row, + ) + + await loop.run_in_executor(None, _upsert) + + # Kick the off-platform file-host downloader for any links this run + # recorded (mega/gdrive/…). Global + idempotent (only claims pending/ + # retryable rows); the beat sweep is the backstop. Lazy import dodges a + # task-module import cycle. + from ..tasks.external import sweep_external_links + sweep_external_links.delay() + ev = (await self.async_session.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() diff --git a/backend/app/services/external_fetch.py b/backend/app/services/external_fetch.py new file mode 100644 index 0000000..265092b --- /dev/null +++ b/backend/app/services/external_fetch.py @@ -0,0 +1,248 @@ +"""Fetchers for off-platform file hosts (mega / gdrive / mediafire / dropbox / +pixeldrain). + +A shared, reusable subsystem: given an external_link URL, fetch the file(s) into +a destination directory and report the outcome. The download worker (separate +slice) drives these off the external_link ledger; any in-house downloader can +call `fetch_external()` directly. + +No single tool covers all five hosts, so a small registry maps host → fetch +function behind one signature: + + fetch_external(host, url, dest_dir, *, timeout, should_stop) -> FetchResult + +Backends: + - dropbox : force the direct-download variant (dl=1) + stream GET. + - pixeldrain : GET the /api/file/{id} endpoint. + - mediafire : scrape the download page for the direct link + stream GET. + - gdrive : gdown (handles the confirm-token + virus-scan interstitial). + - mega : `megatools dl` subprocess (public link incl. #key); needs the + `megatools` binary in the runtime image (Debian apt package). + +Public links work credential-free (rule 26); per-host creds are a later Settings +concern. Plain-HTTP homelab — no secure-context API. The HTTP / gdown / +subprocess calls go through module-level seams so unit tests run without +network, gdown, or MEGAcmd. +""" +from __future__ import annotations + +import logging +import os +import re +import subprocess +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import requests + +log = logging.getLogger(__name__) + +_CHUNK = 1 << 16 +_DEFAULT_TIMEOUT = 600.0 +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +) +# MediaFire's download page embeds the real file URL in a download-button href. +_MEDIAFIRE_RE = re.compile( + r'href="(https://download[^"]+?\.mediafire\.com/[^"]+)"', re.IGNORECASE +) +_CD_FILENAME_RE = re.compile(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', re.IGNORECASE) + + +@dataclass +class FetchResult: + files: list[Path] = field(default_factory=list) + bytes: int = 0 + error: str | None = None + + @property + def ok(self) -> bool: + return self.error is None and bool(self.files) + + +class ExternalFetchError(Exception): + """A fetch failed in a way worth recording on the link's last_error.""" + + +# -- seams (monkeypatched in tests) ---------------------------------------- + +def _http_get(url: str, *, timeout: float, headers: dict | None = None, + stream: bool = True) -> requests.Response: + hdrs = {"User-Agent": _USER_AGENT} + if headers: + hdrs.update(headers) + return requests.get(url, timeout=timeout, headers=hdrs, stream=stream) + + +def _gdown_download(url: str, out_dir: str) -> str | None: + """Download a Google Drive url into out_dir via gdown; return the written + path. Imported lazily so the dep is optional at module-import time.""" + # Lazy import: keeps gdown an optional dep that isn't needed to import this + # module (e.g. the no-DB unit lane that never exercises a real fetch). + import gdown + # A trailing sep tells gdown to keep the server-side filename inside out_dir. + return gdown.download(url, output=out_dir + os.sep, quiet=True, fuzzy=True) + + +def _run_mega_get(url: str, out_dir: str, *, timeout: float) -> None: + """Download a mega.nz public link (key in the #fragment) into out_dir via + `megatools dl` (the Debian `megatools` package). Raises ExternalFetchError + on non-zero exit.""" + # Fixed argv (not shell): only `url` is external input, passed positionally, + # so there's no shell-injection surface. + proc = subprocess.run( + ["megatools", "dl", "--path", out_dir, url], + capture_output=True, text=True, timeout=timeout, check=False, + ) + if proc.returncode != 0: + raise ExternalFetchError( + f"megatools dl exit {proc.returncode}: {(proc.stderr or '').strip()[:300]}" + ) + + +# -- helpers --------------------------------------------------------------- + +def _safe_name(name: str, fallback: str) -> str: + name = os.path.basename((name or "").strip().strip('"')) + # Strip path separators / control chars; never empty. + name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", name).strip(". ") + return name or fallback + + +def _filename_from(resp: requests.Response, url: str, fallback: str) -> str: + cd = resp.headers.get("Content-Disposition", "") + m = _CD_FILENAME_RE.search(cd) + if m: + return _safe_name(m.group(1), fallback) + path_name = os.path.basename(urlsplit(url).path) + return _safe_name(path_name, fallback) + + +def _stream_to_file(resp: requests.Response, dest: Path, + should_stop: Callable[[], bool]) -> int: + """Stream a response body to `dest` (atomic via .part). Returns byte count. + Honors should_stop between chunks (partial file removed).""" + part = dest.with_name(dest.name + ".part") + total = 0 + try: + with part.open("wb") as fh: + for chunk in resp.iter_content(chunk_size=_CHUNK): + if should_stop(): + raise ExternalFetchError("stopped") + if chunk: + fh.write(chunk) + total += len(chunk) + except BaseException: + part.unlink(missing_ok=True) + raise + os.replace(part, dest) + return total + + +def _get_to_dir(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool], fallback: str, + headers: dict | None = None) -> FetchResult: + resp = _http_get(url, timeout=timeout, headers=headers, stream=True) + if resp.status_code != 200: + return FetchResult(error=f"HTTP {resp.status_code} for {url}") + name = _filename_from(resp, url, fallback) + dest = dest_dir / name + written = _stream_to_file(resp, dest, should_stop) + return FetchResult(files=[dest], bytes=written) + + +# -- per-host fetchers ----------------------------------------------------- + +def _fetch_dropbox(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool]) -> FetchResult: + # Force the direct-download variant: dl=1 (Dropbox serves an HTML preview + # for dl=0). Rewrite/insert the param rather than string-replace so ?dl=0, + # &dl=0, and a missing param all resolve. + parts = urlsplit(url) + q = dict(parse_qsl(parts.query)) + q["dl"] = "1" + direct = urlunsplit(parts._replace(query=urlencode(q))) + return _get_to_dir(direct, dest_dir, timeout=timeout, + should_stop=should_stop, fallback="dropbox-file") + + +def _fetch_pixeldrain(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool]) -> FetchResult: + # /u/{id} (and /l/{id}) → the API file endpoint. + file_id = urlsplit(url).path.rstrip("/").split("/")[-1] + if not file_id: + return FetchResult(error=f"no pixeldrain id in {url}") + api = f"https://pixeldrain.com/api/file/{file_id}" + return _get_to_dir(api, dest_dir, timeout=timeout, + should_stop=should_stop, fallback=f"{file_id}.bin") + + +def _fetch_mediafire(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool]) -> FetchResult: + page = _http_get(url, timeout=timeout, stream=False) + if page.status_code != 200: + return FetchResult(error=f"HTTP {page.status_code} for mediafire page") + m = _MEDIAFIRE_RE.search(page.text or "") + if not m: + return FetchResult(error="mediafire direct link not found on page") + return _get_to_dir(m.group(1), dest_dir, timeout=timeout, + should_stop=should_stop, fallback="mediafire-file") + + +def _fetch_gdrive(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool]) -> FetchResult: + out = _gdown_download(url, str(dest_dir)) + if not out: + return FetchResult(error="gdown returned no file (quota / private?)") + p = Path(out) + if not p.exists(): + return FetchResult(error=f"gdown reported {out} but it is missing") + return FetchResult(files=[p], bytes=p.stat().st_size) + + +def _fetch_mega(url: str, dest_dir: Path, *, timeout: float, + should_stop: Callable[[], bool]) -> FetchResult: + before = set(dest_dir.iterdir()) if dest_dir.exists() else set() + _run_mega_get(url, str(dest_dir), timeout=timeout) + new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()] + if not new: + return FetchResult(error="mega-get wrote no new file") + return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new)) + + +_REGISTRY: dict[str, Callable[..., FetchResult]] = { + "dropbox": _fetch_dropbox, + "pixeldrain": _fetch_pixeldrain, + "mediafire": _fetch_mediafire, + "gdrive": _fetch_gdrive, + "mega": _fetch_mega, +} + +SUPPORTED_HOSTS = tuple(_REGISTRY) + + +def fetch_external(host: str, url: str, dest_dir: Path, *, + timeout: float = _DEFAULT_TIMEOUT, + should_stop: Callable[[], bool] = lambda: False) -> FetchResult: + """Fetch `url` (a `host` link) into `dest_dir`. Returns a FetchResult; never + raises — any backend error (transport, non-200, scrape miss, subprocess + failure, stop) is captured on `.error` so the worker can record it and move + on.""" + fetcher = _REGISTRY.get(host) + if fetcher is None: + return FetchResult(error=f"unsupported host {host!r}") + dest_dir.mkdir(parents=True, exist_ok=True) + try: + return fetcher(url, dest_dir, timeout=timeout, should_stop=should_stop) + except requests.RequestException as exc: + return FetchResult(error=f"transport error: {exc}") + except subprocess.TimeoutExpired: + return FetchResult(error="timed out") + except ExternalFetchError as exc: + return FetchResult(error=str(exc)) + except Exception as exc: # never let a backend quirk kill the worker + log.warning("external fetch (%s) failed for %s: %s", host, url, exc) + return FetchResult(error=f"{type(exc).__name__}: {exc}") diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 3add0e1..274ba86 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -147,6 +147,12 @@ class DownloadResult: files_quarantined: int = 0 quarantined_paths: list[str] = field(default_factory=list) written_paths: list[str] = field(default_factory=list) + # Native ingester only: post-only sidecar paths for posts that have NO + # downloadable media (pure-text posts), so the importer can still upsert the + # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via + # Importer.upsert_post_record (keyed on external_post_id → updates, never + # doubles, the same Post a media import would create). + post_record_paths: list[str] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index d5d2185..c9b5abe 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -23,6 +23,7 @@ from sqlalchemy.orm import Session from ..models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, ImportSettings, @@ -34,6 +35,7 @@ from ..utils import safe_probe from ..utils.paths import ( derive_subdir, derive_top_level_artist, + filehash_from_url, hash_suffixed_name, safe_ext, ) @@ -42,6 +44,7 @@ from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify from .archive_extractor import extract_archive, is_archive from .attachment_store import AttachmentStore +from .link_extract import extract_external_links from .thumbnailer import Thumbnailer log = logging.getLogger(__name__) @@ -305,6 +308,32 @@ class Importer: ), ) + def _sync_external_links(self, post: Post) -> None: + """Record off-platform file-host links (mega/gdrive/mediafire/dropbox/ + pixeldrain) found in the post body, so they're never silently dropped + and the download worker can fetch them later. Shared by every platform's + import (runs off Post.description). INSERT-MISSING only — an existing row + keeps its status/attempts (a re-import must not reset a link already + downloaded or dead-lettered). Identity is (post_id, url); the full url + incl. #fragment is preserved by the extractor.""" + links = extract_external_links(post.description) + if not links: + return + self.session.flush() # ensure post.id is assigned before we reference it + existing = set(self.session.execute( + select(ExternalLink.url).where(ExternalLink.post_id == post.id) + ).scalars().all()) + for link in links: + if link.url in existing: + continue + self.session.add(ExternalLink( + post_id=post.id, + artist_id=post.artist_id, + host=link.host, + url=link.url, + label=link.label, + )) + def import_one(self, source: Path) -> ImportResult: """Dispatch by kind. Media → normal pipeline. Archive → extract media members (one Post via the archive-adjacent sidecar) and @@ -710,6 +739,67 @@ class Importer: self.session.commit() return ImportResult(status="refreshed", image_id=existing.id) + def upsert_post_record( + self, sidecar: Path, *, artist: Artist | None = None, + source: Source | None = None, + ) -> bool: + """Upsert the Post for a post-ONLY sidecar (a media-less post), so the + artist archive includes text posts (their body + external links). + + Reuses `_find_or_create_post` (keyed on external_post_id) so it UPDATES + the SAME Post a media import would create — never doubles. Fields are + FILLED, never clobbered with empty: parse_sidecar yields None for empty + values, and a None field is left untouched (an empty feed body never + wipes a populated one). Returns True if a Post was upserted, False if the + sidecar was unusable (parse failure / no artist).""" + try: + data = json.loads(sidecar.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("post-record sidecar parse failed for %s: %s", sidecar, exc) + return False + + sd = parse_sidecar(data) + + if artist is None: + name = self._sidecar_artist_name(data) + artist = self._upsert_artist(name) if name else None + if artist is None: + log.warning("post-record sidecar %s has no artist; skipping", sidecar) + return False + + if source is not None: + src = source + else: + platform = sd.platform or "unknown" + src = self._lookup_source_for_sidecar( + artist_id=artist.id, platform=platform, + ) + + epid = sd.external_post_id or sidecar.stem + post = self._find_or_create_post( + source_id=src.id if src else None, + external_post_id=epid, + artist_id=artist.id, + ) + if post.artist_id is None: + post.artist_id = artist.id + if sd.post_url is not None: + post.post_url = sd.post_url + if sd.post_title is not None: + post.post_title = sd.post_title + if sd.post_date is not None: + post.post_date = sd.post_date + if sd.description is not None: + post.description = sd.description + if sd.attachment_count is not None: + post.attachment_count = sd.attachment_count + post.raw_metadata = sd.raw + self._sync_external_links(post) + self.session.commit() + return True + def attach_in_place( self, path: Path, @@ -957,6 +1047,14 @@ class Importer: if record.artist_id is None: record.artist_id = artist.id + # #830 Phase 2: persist the file's CDN source identity (NULL-only, so a + # re-import / enrich-on-duplicate never clobbers it) — the filehash is + # the join key that lets post_feed_service remap the body's inline + # `` to this local copy at render time. + if sd.source_url and record.source_filehash is None: + record.source_url = sd.source_url + record.source_filehash = filehash_from_url(sd.source_url) + if explicit_source is not None: src = explicit_source else: @@ -982,6 +1080,7 @@ class Importer: if sd.attachment_count is not None: post.attachment_count = sd.attachment_count post.raw_metadata = sd.raw + self._sync_external_links(post) # Race-safe (image_record_id, post_id) upsert — mirrors the # _find_or_create_source/post savepoint pattern. The plain diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 7de059c..d83525f 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -118,10 +118,16 @@ class Ingester: # tick has no resumable backfill state. checkpoint = mode in ("backfill", "recovery") ledger_key = self._ledger_key + # Optional seams (Patreon native ingester): capture pure-text posts that + # have NO downloadable media so the artist archive is complete. Absent on + # stub clients/downloaders (unit tests) → media-less posts skipped as before. + post_record_key = getattr(self.client, "post_record_key", None) + write_post_record = getattr(self.downloader, "write_post_record", None) start = time.monotonic() last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] written: list[str] = [] + post_records: list[str] = [] quarantined_paths: list[str] = [] downloaded = 0 errors = 0 @@ -158,6 +164,7 @@ class Ingester: files_quarantined=quarantined, quarantined_paths=list(quarantined_paths), written_paths=written, + post_record_paths=list(post_records), stdout="\n".join(log_lines), stderr="", return_code=return_code, @@ -220,6 +227,29 @@ class Ingester: # resume_cursor None, so everything counts. if not (resume_cursor and page_cursor == resume_cursor): chunk_new_posts += 1 + # Capture the post body + external links ONCE per post (gated by + # the synthetic post key in the seen-ledger), for EVERY post — + # whether or not it has downloadable media. This is what makes a + # backfill/recovery re-walk RECAPTURE bodies + links for posts + # whose media is already on disk: re-downloading existing media + # never fills links the system never had, so the body recapture + # has to ride the walk itself. Detail-fetch (for an empty feed + # body) happens at most once per post — the gate then spares it on + # later walks. bypass_seen (recovery) re-captures unconditionally. + if post_record_key and write_post_record: + rk = post_record_key(post) + if rk is not None: + pkey, ppid = rk + already = ( + set() if bypass_seen + else self._seen_keys(source_id, [pkey]) + ) + if pkey not in already: + rec_path = write_post_record(post, artist_slug) + if rec_path is not None: + post_records.append(str(rec_path)) + self._mark_seen(source_id, [(pkey, ppid)]) + media = self.client.extract_media(post, included) if not media: continue diff --git a/backend/app/services/link_extract.py b/backend/app/services/link_extract.py new file mode 100644 index 0000000..e6ac046 --- /dev/null +++ b/backend/app/services/link_extract.py @@ -0,0 +1,105 @@ +"""Extract off-platform file-host links from a post body. + +Pure (no I/O) so it's unit-testable and reusable by EVERY in-house downloader's +import path — every platform stores its body in `Post.description`, so running +this there covers them all. Finds links to the supported external file hosts in +a post's HTML body (both `` anchors and bare URLs in text), +unwraps a Patreon outbound-redirect wrapper, and preserves the FULL url +including the `#fragment` (mega.nz puts the decryption key there) and the query +string — without those a mega download is impossible. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from html import unescape +from urllib.parse import parse_qs, urlsplit + +# Supported file-host enum values (kept in sync with the external_link CHECK). +SUPPORTED_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") + +# Bare-domain suffix → canonical host label. Matched against the URL netloc +# (exact or as a dotted suffix, so www./dl. subdomains resolve too). +_HOST_MAP = { + "mega.nz": "mega", + "mega.co.nz": "mega", + "drive.google.com": "gdrive", + "mediafire.com": "mediafire", + "dropbox.com": "dropbox", + "pixeldrain.com": "pixeldrain", +} + +# `label` — DOTALL so a label spanning tags/newlines is caught. +_HREF_RE = re.compile( + r"""]*?\bhref=["']([^"']+)["'][^>]*>(.*?)""", + re.IGNORECASE | re.DOTALL, +) +_TAG_RE = re.compile(r"<[^>]+>") +# Bare http(s) URL in text. Stops at whitespace, quotes, angle brackets, and +# closing brackets — but KEEPS `#`, `&`, `?`, `=` so fragments/queries survive. +_URL_RE = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE) + + +@dataclass(frozen=True) +class ExtractedLink: + host: str # one of SUPPORTED_HOSTS + url: str # full url incl. query + #fragment + label: str | None # visible anchor text, when present + + +def _netloc(url: str) -> str: + # Lowercase host without credentials or port. + return urlsplit(url).netloc.lower().split("@")[-1].split(":")[0] + + +def host_for(url: str) -> str | None: + """Canonical host label for a url, or None if it's not a supported host.""" + netloc = _netloc(url) + for suffix, host in _HOST_MAP.items(): + if netloc == suffix or netloc.endswith("." + suffix): + return host + return None + + +def _unwrap(url: str) -> str: + """Unwrap a Patreon outbound-redirect wrapper to its inner target, so a + wrapped mega/gdrive link resolves to the real host. Patreon has used both + `www.patreon.com/...?url=` and the `l.patreon.com` shim; check the + common target-param names. Non-Patreon urls pass through untouched.""" + netloc = _netloc(url) + if not (netloc == "patreon.com" or netloc.endswith(".patreon.com")): + return url + qs = parse_qs(urlsplit(url).query) + for key in ("url", "u", "ext_url", "redirect", "target"): + vals = qs.get(key) + if vals and vals[0]: + return unescape(vals[0]).strip() + return url + + +def extract_external_links(html: str | None) -> list[ExtractedLink]: + """All supported-host links in `html`, de-duplicated by url (first wins, so + an anchor's label is kept over a later bare sighting of the same url).""" + if not html: + return [] + found: dict[str, ExtractedLink] = {} + + # 1) Anchors first — they carry a human label ("Mega - Streamable"). + for raw_href, inner in _HREF_RE.findall(html): + url = _unwrap(unescape(raw_href).strip()) + host = host_for(url) + if host is None: + continue + label = unescape(_TAG_RE.sub("", inner)).strip() or None + found.setdefault(url, ExtractedLink(host=host, url=url, label=label)) + + # 2) Bare URLs pasted in text (no label). Trailing prose punctuation is + # trimmed; the href values already captured above de-dup away here. + for raw in _URL_RE.findall(html): + url = _unwrap(unescape(raw).strip().rstrip(".,;")) + host = host_for(url) + if host is None: + continue + found.setdefault(url, ExtractedLink(host=host, url=url, label=None)) + + return list(found.values()) diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index b896c4b..f79f627 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit import requests -from ..utils.paths import safe_ext +from ..utils.paths import filehash_from_url, safe_ext log = logging.getLogger(__name__) @@ -90,12 +90,6 @@ _FIELDS_POST = ( _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" _FIELDS_CAMPAIGN = "name,url" -# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is -# Patreon's stable per-file identity and is what we dedup + ledger against. -# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere -# in the URL (path or query); real Patreon CDN URLs carry exactly one. -_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") - # Inline post `content` is HTML; images are emitted as . # Pull every src; downstream dedup collapses any that duplicate a gallery item # by filehash. Tolerant of attribute ordering and single/double quotes. @@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session: def _filehash(url: str) -> str | None: - if not url: - return None - match = _FILEHASH_RE.search(url) - return match.group(1).lower() if match else None + # Delegate to the shared extractor (utils.paths) so capture-time persistence + # and render-time inline-image matching use the EXACT same identity. + return filehash_from_url(url) def _basename_from_url(url: str) -> str: @@ -526,6 +519,19 @@ class PatreonClient: "date": published if isinstance(published, str) else None, } + @staticmethod + def post_record_key(post: dict) -> tuple[str, str] | None: + """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or + None when the post has no id. The synthetic `post:` key lets the + generic core gate text-post capture through the SAME seen-ledger as media + — so a text post's body is detail-fetched + recorded ONCE, not re-fetched + every tick. Part of the client contract the core uses via getattr.""" + pid = post.get("id") + pid = str(pid) if pid is not None else "" + if not pid: + return None + return (f"post:{pid}", pid) + # -- iteration --------------------------------------------------------- def iter_posts( @@ -562,6 +568,49 @@ class PatreonClient: return current_cursor = next_cursor + # -- detail (full body enrichment) ------------------------------------- + + def fetch_post_detail_content(self, post_id: str) -> str | None: + """Best-effort fetch of a post's full HTML `content` from the per-post + DETAIL endpoint (`/api/posts/{id}`). + + The feed/list endpoint (`/api/posts`) frequently returns `content` as + null even though we request it — the full body (its formatting, inline + ``, and external `` links) only comes back from the + single-post detail resource. The downloader calls this to enrich a post + whose feed body was empty before writing the importer sidecar. + + Best-effort BY DESIGN: the media download is the primary job, so a body + we can't fetch must never fail the walk. Every failure path (no id, + transport error, non-200, non-JSON, missing/empty content) returns None + rather than raising — distinct from the loud drift/auth raises on the + feed path, which gate real downloads. + """ + if not post_id: + return None + if self._request_sleep > 0: + time.sleep(self._request_sleep) # pace the API endpoint (plan #703) + url = f"{_POSTS_URL}/{post_id}" + params = {"fields[post]": "content", "json-api-version": "1.0"} + try: + resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS) + except requests.RequestException as exc: + log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc) + return None + if resp.status_code != 200: + log.warning( + "Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id + ) + return None + try: + payload = resp.json() + except ValueError: + return None + data = payload.get("data") if isinstance(payload, dict) else None + attrs = data.get("attributes") if isinstance(data, dict) else None + content = attrs.get("content") if isinstance(attrs, dict) else None + return content if isinstance(content, str) and content.strip() else None + # -- verify ------------------------------------------------------------ def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 7abc2df..1f7443d 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -168,10 +168,17 @@ class PatreonDownloader: validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, + content_fetcher: Callable[[str], str | None] | None = None, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self._validate = validate + # Best-effort enrichment seam: (post_id) -> full HTML body, or None. The + # feed endpoint often omits `content`; the adapter wires this to + # PatreonClient.fetch_post_detail_content so the sidecar captures the + # real body (formatting + inline + external links). + # None in unit tests / when enrichment isn't wanted. + self._content_fetcher = content_fetcher # Politeness: seconds to sleep before each actual media download (paces # the CDN; honors ImportSettings.download_rate_limit_seconds, the same # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing. @@ -287,7 +294,7 @@ class PatreonDownloader: path=quarantine_dest, error=reason, ) - self._write_sidecar(post, out_path) + self._write_sidecar(post, out_path, source_url=media.url) return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) # -- download seams ---------------------------------------------------- @@ -483,7 +490,9 @@ class PatreonDownloader: # -- sidecar ----------------------------------------------------------- - def _write_sidecar(self, post: dict, media_path: Path) -> Path: + def _write_sidecar( + self, post: dict, media_path: Path, *, source_url: str | None = None + ) -> Path: """Write the importer-consumed sidecar next to `media_path`. Patreon uses base-default sidecar keys (parse_sidecar maps @@ -491,10 +500,37 @@ class PatreonDownloader: content->description, published_at->post_date, url->post_url). Patreon registers no derive_post_url, so `url` is trusted as the permalink — we pass the post's attributes.url. + + `source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists + its filehash on the ImageRecord so the post body's inline `` can + be remapped to the local copy at render time. """ + return self._write_sidecar_data( + post, media_path.with_suffix(".json"), source_url=source_url + ) + + def _write_sidecar_data( + self, post: dict, sidecar_path: Path, *, source_url: str | None = None + ) -> Path: + """Serialize the post's metadata to `sidecar_path`. Shared by the + per-media sidecar (next to each file) and the post-only sidecar + (`write_post_record`, for media-less posts). `source_url` is set only for + the per-media sidecar — a media-less post has no source file.""" attrs = post.get("attributes") or {} title = attrs.get("title") content = attrs.get("content") + # The feed/list endpoint frequently returns an empty `content`; the full + # HTML body (formatting + inline + external links) only + # comes from the per-post detail endpoint. Enrich on first write for this + # post and MEMOIZE by mutating the shared `post` dict — so a multi-image + # post fetches detail exactly once, and a fully-seen post (no fresh + # download → no sidecar write) never pays the extra GET. + if (not isinstance(content, str) or not content.strip()) and self._content_fetcher: + fetched = self._content_fetcher(str(post.get("id") or "")) + if fetched: + attrs["content"] = fetched + post["attributes"] = attrs + content = fetched published = attrs.get("published_at") url = attrs.get("url") data = { @@ -505,6 +541,20 @@ class PatreonDownloader: "published_at": published if isinstance(published, str) else None, "url": url if isinstance(url, str) else None, } - sidecar_path = media_path.with_suffix(".json") + if source_url: + data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path + + def write_post_record(self, post: dict, artist_slug: str) -> Path | None: + """Write a post-ONLY sidecar (no media file) for a media-less post, so + the importer can still upsert the Post + its body — text posts often hold + the only copy of an external link. Named `_post.json`: the + leading underscore keeps it from colliding with a media sidecar + (`_.json`) and from being resolved as some media file's sidecar + by find_sidecar. Returns the path, or None when the post has no id.""" + if not str(post.get("id") or ""): + return None + post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) + post_dir.mkdir(parents=True, exist_ok=True) + return self._write_sidecar_data(post, post_dir / "_post.json") diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index be2c973..c4e8d73 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -112,7 +112,10 @@ class PatreonIngester(Ingester): downloader if downloader is not None else PatreonDownloader( - self.images_root, cookies_path, validate=validate, rate_limit=rate_limit + self.images_root, cookies_path, validate=validate, rate_limit=rate_limit, + # Enrich empty feed bodies from the per-post detail endpoint, via + # the SAME client (shares its cookie session + request pacing). + content_fetcher=resolved_client.fetch_post_detail_content, ) ) super().__init__( diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index f83d0f0..d21f19d 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -11,19 +11,28 @@ attachments from PostAttachment) so the API layer can jsonify directly. """ from __future__ import annotations +from html import unescape + from sqlalchemy import and_, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from ..models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, Post, PostAttachment, Source, ) +from ..utils.html_sanitize import ( + extract_img_srcs, + rewrite_img_srcs, + sanitize_post_html, +) +from ..utils.paths import filehash_from_url from ..utils.text import html_to_plain, truncate_at_word -from .gallery_service import thumbnail_url +from .gallery_service import image_url, thumbnail_url from .pagination import decode_cursor, encode_cursor DESCRIPTION_LIMIT = 280 @@ -191,8 +200,71 @@ class PostFeedService: atts_map = await self._attachments_for([post.id]) item = self._to_dict(post, artist, source, thumbs_map, atts_map) item["description_full"] = html_to_plain(post.description) + # Sanitized HTML body for faithful (semantic) rendering in the post view; + # detail-only (the feed list stays lightweight plain text). None when the + # post has no body. Inline `` sources are remapped to locally-served + # copies (#830 Phase 2) so the body never hotlinks the public CDN. + item["description_html"] = await self._localize_inline_images( + sanitize_post_html(post.description), post.artist_id, + ) + item["external_links"] = await self._external_links_for(post.id) return item + async def _localize_inline_images( + self, html: str | None, artist_id: int | None, + ) -> str | None: + """Rewrite a post body's inline `` to locally-served copies. + + The join key is the CDN filehash the downloader persisted on each + ImageRecord (source_filehash): for every body image whose filehash maps + to a stored image of THIS artist, swap the src to /images/. Images + we never captured (or pre-Phase-2 rows with no filehash) are left as-is — + they keep hotlinking, which is the prior behavior. Scoped to the post's + artist so one creator's body never resolves to another's file.""" + if not html or artist_id is None: + return html + srcs = extract_img_srcs(html) + if not srcs: + return html + # filehash -> the raw (as-in-HTML) src strings carrying it. A body can + # repeat the same image; keep every raw form so each is substituted. + by_hash: dict[str, list[str]] = {} + for raw in srcs: + fh = filehash_from_url(unescape(raw)) + if fh: + by_hash.setdefault(fh, []).append(raw) + if not by_hash: + return html + rows = (await self.session.execute( + select(ImageRecord.source_filehash, ImageRecord.path) + .where( + ImageRecord.artist_id == artist_id, + ImageRecord.source_filehash.in_(list(by_hash)), + ) + )).all() + replace: dict[str, str] = {} + for fh, path in rows: + for raw in by_hash.get(fh, ()): + replace[raw] = image_url(path) + return rewrite_img_srcs(html, replace) + + async def _external_links_for(self, post_id: int) -> list[dict]: + """Off-platform file-host links recorded for a post (detail-only). Each + carries its host, full url, label, and download status so the post view + can surface them (and, later, a retry/download affordance).""" + rows = (await self.session.execute( + select(ExternalLink) + .where(ExternalLink.post_id == post_id) + .order_by(ExternalLink.id.asc()) + )).scalars().all() + return [ + { + "id": e.id, "host": e.host, "url": e.url, + "label": e.label, "status": e.status, + } + for e in rows + ] + # --- composition helpers --------------------------------------------- async def _thumbnails_for( diff --git a/backend/app/tasks/external.py b/backend/app/tasks/external.py new file mode 100644 index 0000000..e5f243d --- /dev/null +++ b/backend/app/tasks/external.py @@ -0,0 +1,356 @@ +"""Celery worker for off-platform file-host downloads (external_link ledger). + +Walks the external_link table and fetches each pending/retryable link via +external_fetch, then routes the downloaded file(s) through the existing importer +(attach_in_place) so an archive becomes ImageRecords and any other file a +PostAttachment — linked to the SAME post the link came from (via a synthesized +sidecar). Long-running-task hygiene (rule #89): per-fetch wall-clock timeout, +attempt tracking + dead-letter, a recovery sweep, and retention of dead rows. + +Concurrency: a per-host Redis lock serializes fetches to one-per-host (mega / +gdrive ban-avoidance), and an atomic status claim (pending/failed → +downloading) stops two workers grabbing the same link. + +Files land in the artist's library tree (so an attached art image stays in +place, mirroring the gallery-dl download path); a captured archive/attachment is +copied into its store and the on-disk original removed. +""" +from __future__ import annotations + +import json +import logging +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import redis +from sqlalchemy import delete, select, update + +from ..celery_app import celery +from ..config import get_config +from ..models import Artist, ExternalLink, ImportSettings, Post, Source +from ..services.external_fetch import fetch_external +from ..services.importer import Importer +from ..services.thumbnailer import Thumbnailer +from ._sync_engine import sync_session_factory as _sync_session_factory + +log = logging.getLogger(__name__) + +IMAGES_ROOT = Path("/images") + +# After this many failed attempts a link is dead-lettered (skipped by routine +# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger. +DEAD_LETTER_THRESHOLD = 3 +# Per-fetch wall-clock budget — films/packs are large; the celery time_limit is +# the backstop above this. +_FETCH_TIMEOUT = 3000.0 +# Links enqueued per sweep — bounds the burst when a big backfill records many. +_SWEEP_BATCH = 50 +# Dead rows older than this are pruned (retention). +_RETENTION_DAYS = 30 +# Per-host serialize lock. +_LOCK_PREFIX = "fc:extdl_lock:" +_LOCK_TTL = 3600 +_SERIALIZE_COUNTDOWN = 120 +_MAX_SERIALIZE_WAITS = 30 + +_redis_client: redis.Redis | None = None + + +def _redis() -> redis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = redis.from_url(get_config().celery_broker_url) + return _redis_client + + +def _host_enabled(settings: ImportSettings, host: str) -> bool: + """Per-host enable flag — defaults True (rule #26: works out of the box). + The Settings UI slice adds real columns; getattr keeps this forward- + compatible without a migration in this slice.""" + return bool(getattr(settings, f"extdl_{host}_enabled", True)) + + +def _write_link_sidecar(file: Path, post: Post, platform: str, artist: Artist) -> None: + """Sidecar next to a fetched file so the importer links it to `post` + (category+id resolve the source+post; matches gallery-dl's sidecar shape).""" + data = { + "category": platform, + "id": post.external_post_id, + "url": post.post_url, + "title": post.post_title, + "artist": artist.name, + } + Path(str(file) + ".json").write_text(json.dumps(data)) + + +@celery.task( + name="backend.app.tasks.external.fetch_external_link", + bind=True, + soft_time_limit=3300, + time_limit=3600, +) +def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict: + """Fetch one external_link and route its file(s) through the importer.""" + SessionLocal = _sync_session_factory() + + # Atomic claim: only an actionable (pending/failed) row transitions to + # downloading, so a duplicate enqueue (sweep + post-download hook) no-ops. + with SessionLocal() as session: + claimed = session.execute( + update(ExternalLink) + .where( + ExternalLink.id == link_id, + ExternalLink.status.in_(("pending", "failed")), + ) + .values(status="downloading") + .returning(ExternalLink.id) + ).first() + session.commit() + if claimed is None: + return {"link_id": link_id, "skipped": "not claimable"} + + link = session.get(ExternalLink, link_id) + post = session.get(Post, link.post_id) if link.post_id else None + if post is None: + link.status = "dead" + link.last_error = "post missing" + link.completed_at = datetime.now(UTC) + session.commit() + return {"link_id": link_id, "error": "post missing"} + artist = session.get(Artist, post.artist_id) + source = session.get(Source, post.source_id) if post.source_id else None + platform = source.platform if source is not None else "patreon" + settings = ImportSettings.load_sync(session) + + if not _host_enabled(settings, link.host): + log.info("extdl skip (host disabled): link=%s host=%s", link_id, link.host) + link.status = "skipped" + link.last_error = f"{link.host} disabled" + link.completed_at = datetime.now(UTC) + session.commit() + return {"link_id": link_id, "skipped": "host disabled"} + + host, url, attempts = link.host, link.url, link.attempts + log.info( + "extdl start: link=%s host=%s post=%s artist=%s attempt=%d url=%s", + link_id, host, post.id, artist.slug, attempts + 1, url, + ) + + # Per-host serialize: one fetch per host at a time. If busy, requeue. + lock = None + try: + lock = _redis().lock(f"{_LOCK_PREFIX}{host}", timeout=_LOCK_TTL, blocking=False) + got = bool(lock.acquire(blocking=False)) + except redis.RedisError as exc: + log.warning("extdl lock unavailable for %s: %s — running uncapped", host, exc) + lock, got = None, True + if not got: + # Release the claim back to pending so a later run can pick it up, then + # requeue with a countdown (bounded, like download_source). + with SessionLocal() as session: + session.execute( + update(ExternalLink).where(ExternalLink.id == link_id) + .values(status="pending") + ) + session.commit() + log.info( + "extdl requeue (host %s busy): link=%s wait=%d", host, link_id, _serialize_waits, + ) + if _serialize_waits < _MAX_SERIALIZE_WAITS: + fetch_external_link.apply_async( + (link_id,), {"_serialize_waits": _serialize_waits + 1}, + countdown=_SERIALIZE_COUNTDOWN, + ) + else: + log.warning("extdl giving up requeue (host %s busy): link=%s", host, link_id) + return {"link_id": link_id, "requeued": "host busy"} + + started = time.monotonic() + post_dir = IMAGES_ROOT / artist.slug / platform / "external" / str(post.external_post_id) + try: + result = fetch_external(host, url, post_dir, timeout=_FETCH_TIMEOUT) + with SessionLocal() as session: + link = session.get(ExternalLink, link_id) + if not result.ok: + log.warning( + "extdl fetch failed: link=%s host=%s error=%s", + link_id, host, result.error, + ) + _record_failure(link, attempts, result.error or "fetch failed") + session.commit() + return {"link_id": link_id, "error": result.error} + + log.info( + "extdl fetched: link=%s host=%s files=%d bytes=%d", + link_id, host, len(result.files), result.bytes, + ) + # Re-load post/artist/source attached to THIS session before handing + # them to the importer (the claim block's instances are detached). + post = session.get(Post, link.post_id) + artist = session.get(Artist, post.artist_id) + source = session.get(Source, post.source_id) if post.source_id else None + image_ids = _route_files(session, result.files, post, platform, artist, source) + link.status = "downloaded" + link.last_error = None + link.completed_at = datetime.now(UTC) + link.duration_seconds = time.monotonic() - started + session.commit() + log.info( + "extdl done: link=%s host=%s post=%s files=%d image(s)=%d dur=%.1fs", + link_id, host, post.id, len(result.files), len(image_ids), + link.duration_seconds, + ) + + # Thumbnails + ML for any newly-attached images (mirrors the download + # path). Lazy import to dodge a task-module import cycle. + if image_ids: + from .ml import tag_and_embed + from .thumbnail import generate_thumbnail + for img_id in image_ids: + generate_thumbnail.delay(img_id) + tag_and_embed.delay(img_id) + return {"link_id": link_id, "files": len(result.files), "images": len(image_ids)} + except Exception as exc: # never leave a link stuck in 'downloading' + log.exception("external fetch task failed for link %s", link_id) + with SessionLocal() as session: + link = session.get(ExternalLink, link_id) + if link is not None and link.status == "downloading": + _record_failure(link, attempts, f"{type(exc).__name__}: {exc}") + session.commit() + raise + finally: + if lock is not None: + try: + lock.release() + except Exception: # noqa: BLE001 — TTL may have already freed it + pass + + +def _record_failure(link: ExternalLink, attempts: int, error: str) -> None: + nxt = attempts + 1 + link.attempts = nxt + link.last_error = error[:1000] + link.status = "dead" if nxt >= DEAD_LETTER_THRESHOLD else "failed" + link.completed_at = datetime.now(UTC) + log.warning( + "extdl link %s -> %s (attempt %d/%d): %s", + link.id, link.status, nxt, DEAD_LETTER_THRESHOLD, error[:200], + ) + + +def _route_files(session, files, post, platform, artist, source) -> list[int]: + """Import each fetched file through the SAME pipeline extracted-zip members + use — importer.attach_in_place — so a downloaded archive is extracted to + ImageRecords and provenance-linked to the post (via the synthesized sidecar), + a non-art file is captured as a PostAttachment, and an art image is attached + in place. Returns new image ids so the caller enqueues thumbnail + ML + (tagging) for them, exactly as download_service does for downloaded media. + + Result handling mirrors download_service._phase3_persist branch-for-branch + (duplicate cleanup, the unextracted-archive warning of #718) and logs every + decision — this path is new and the operator expects to iterate on it.""" + settings = ImportSettings.load_sync(session) + importer = Importer( + session=session, + images_root=IMAGES_ROOT, + import_root=IMAGES_ROOT, + thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), + settings=settings, + ) + image_ids: list[int] = [] + for f in files: + _write_link_sidecar(f, post, platform, artist) + result = importer.attach_in_place(f, artist=artist, source=source) + session.commit() + if result.status in ("imported", "superseded"): + ids = list(getattr(result, "member_image_ids", []) or []) + if result.image_id is not None and result.image_id not in ids: + ids.append(result.image_id) + image_ids.extend(ids) + log.info( + "extdl import: post=%s file=%s status=%s -> %d image(s) " + "(provenance-linked, queued for tagging)", + post.id, f.name, result.status, len(ids), + ) + elif result.status == "attached": + # Non-art / archive captured as a PostAttachment (copied into the + # store) — drop the on-disk original. An archive captured WITHOUT + # extracting any image carries the reason (the recurring "zip but no + # images" symptom, #718) — surface it loudly. + if result.error: + log.warning( + "extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s", + post.id, f.name, result.error, + ) + else: + log.info("extdl attachment captured: post=%s file=%s", post.id, f.name) + f.unlink(missing_ok=True) + elif result.status == "skipped": + reason = result.skip_reason.value if result.skip_reason else None + log.info( + "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason, + ) + if reason in ("duplicate_hash", "duplicate_phash"): + f.unlink(missing_ok=True) # canonical copy already in the library + elif result.status == "failed": + log.warning( + "extdl import FAILED: post=%s file=%s error=%s", + post.id, f.name, result.error, + ) + f.unlink(missing_ok=True) + else: # 'refreshed' or any future status — work happened, no new images + log.info( + "extdl import: post=%s file=%s status=%s", post.id, f.name, result.status, + ) + # The synthesized sidecar has done its job — don't litter the tree. + Path(str(f) + ".json").unlink(missing_ok=True) + return image_ids + + +@celery.task(name="backend.app.tasks.external.sweep_external_links") +def sweep_external_links() -> dict: + """Enqueue a bounded batch of actionable links (pending, or failed below the + dead-letter threshold). Driven by the post-download hook and a beat tick.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + ids = session.execute( + select(ExternalLink.id).where( + ExternalLink.status.in_(("pending", "failed")), + ExternalLink.attempts < DEAD_LETTER_THRESHOLD, + ).order_by(ExternalLink.id.asc()).limit(_SWEEP_BATCH) + ).scalars().all() + for link_id in ids: + fetch_external_link.delay(link_id) + return {"enqueued": len(ids)} + + +@celery.task(name="backend.app.tasks.external.recover_external_links") +def recover_external_links() -> dict: + """Recovery sweep (rule #89): reset dead links back to retryable so a stuck + host outage or a since-fixed bug gets another pass, then enqueue. Bounded.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + session.execute( + update(ExternalLink).where(ExternalLink.status == "dead") + .values(status="failed", attempts=0, last_error=None) + ) + session.commit() + return sweep_external_links() + + +@celery.task(name="backend.app.tasks.external.prune_external_links") +def prune_external_links() -> dict: + """Retention (rule #89): delete long-dead links so the ledger doesn't grow + unbounded. Downloaded links are kept (they're the record of what we have).""" + cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS) + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + res = session.execute( + delete(ExternalLink).where( + ExternalLink.status == "dead", + ExternalLink.created_at < cutoff, + ) + ) + session.commit() + return {"pruned": res.rowcount or 0} diff --git a/backend/app/utils/html_sanitize.py b/backend/app/utils/html_sanitize.py index d660a5c..363dd77 100644 --- a/backend/app/utils/html_sanitize.py +++ b/backend/app/utils/html_sanitize.py @@ -6,12 +6,27 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used; bleach is deprecated upstream and intentionally not used. """ +import re + import nh3 +# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped +# post body (headings, lists, quotes, code, inline images, links, line breaks + +# inline emphasis) without layout-injection wrappers. `div`/`span` are +# deliberately NOT allowed — their text is preserved by nh3 and they only carry +# source-site layout/styling we don't want to import (a test pins this). ALLOWED_TAGS = { - "p", "a", "br", "em", "strong", "b", "i", "ul", "ol", "li", "blockquote", + "p", "a", "br", "em", "strong", "b", "i", "u", "s", + "ul", "ol", "li", "blockquote", + "h1", "h2", "h3", "h4", "h5", "h6", "hr", + "pre", "code", "img", "figure", "figcaption", } -ALLOWED_ATTRIBUTES = {"a": {"href", "title"}} +ALLOWED_ATTRIBUTES = { + "a": {"href", "title", "target"}, + "img": {"src", "alt", "title", "width", "height"}, +} +# http/https for hotlinked source images today; relative URLs (e.g. a local +# `/api/...` src once inline images are captured) pass through nh3 untouched. ALLOWED_URL_SCHEMES = {"http", "https"} @@ -37,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None: link_rel="noopener noreferrer", clean_content_tags={"script", "style"}, ) + + +# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough +# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2). +_IMG_SRC_RE = re.compile(r'(]*?\bsrc=")([^"]*)(")', re.IGNORECASE) + + +def extract_img_srcs(html: str | None) -> list[str]: + """The raw `src` values of every in `html`, in document order, deduped + (first kept). Returned EXACTLY as they appear (still entity-escaped) so a + caller can match them back for verbatim substitution.""" + if not html: + return [] + out: list[str] = [] + seen: set[str] = set() + for _pre, src, _post in _IMG_SRC_RE.findall(html): + if src and src not in seen: + seen.add(src) + out.append(src) + return out + + +def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None: + """Swap each whose raw (as-in-HTML) value is a key in `replace` + with the mapped value; others are left untouched. Used to point a post body's + inline images at locally-served copies (#830 Phase 2). Operates only on src + values of already-sanitized markup, so the result stays within the allowlist. + Returns `html` unchanged when it or `replace` is empty.""" + if not html or not replace: + return html + + def _sub(m: re.Match) -> str: + pre, src, post = m.group(1), m.group(2), m.group(3) + return f"{pre}{replace.get(src, src)}{post}" + + return _IMG_SRC_RE.sub(_sub, html) diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py index d9bc48e..7130e90 100644 --- a/backend/app/utils/paths.py +++ b/backend/app/utils/paths.py @@ -1,9 +1,28 @@ """Filesystem path helpers — destination derivation, hash-suffixed names.""" +import re from pathlib import Path _MAX_EXT_LEN = 16 +# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the +# file's stable per-file identity — the same role gallery-dl's `_filehash` +# plays. It is the join key between a post body `` and the local +# copy we downloaded (extract_media dedups content vs gallery images by it), so +# this ONE extractor must be used for both capture-time persistence and +# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex +# run anywhere in the URL (path or query); real CDN URLs carry exactly one. +_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") + + +def filehash_from_url(url: str | None) -> str | None: + """The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None + when the URL is empty / carries no such segment.""" + if not url: + return None + match = _FILEHASH_RE.search(url) + return match.group(1).lower() if match else None + def safe_ext(name: str | Path) -> str: """Conservatively extract a short, alphanumeric file extension. diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 0d81930..1e2a7d5 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -27,6 +27,11 @@ class SidecarData: description: str | None attachment_count: int | None post_date: datetime | None + # Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only + # the per-media sidecar carries it; post-only sidecars leave it None. The + # importer persists it on the ImageRecord so the body's inline `` + # can be matched (by filehash) to the local copy at render time. + source_url: str | None raw: dict @@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData: else: post_url = _first_str(data, ("url", "post_url")) + # Per-file source URL (#830 Phase 2): the native downloader writes it into + # each media sidecar. `url` is the post permalink for Patreon (handled + # above), so source_url has its own dedicated key and never collides. + source_url = _first_str(data, ("source_url",)) + return SidecarData( platform=platform, external_post_id=external_post_id, @@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData: description=description, attachment_count=attachment_count, post_date=post_date, + source_url=source_url, raw=data, ) diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index c38c6da..29f15f9 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -63,8 +63,16 @@ Post {{ post.external_post_id }} + +

{{ descText }}

@@ -79,6 +87,25 @@ @click="toggleDesc" >{{ descExpanded ? 'Show less' : 'Show more' }} + +
+
External files
+
+ {{ lnk.host }} + {{ lnk.label || lnk.url }} + {{ lnk.status }} + +
+
!!props.post.description_plain) const fullDescription = computed(() => detail.value?.description_full || null) +// Backend-sanitized HTML body (detail-only). Drives the faithful render once +// expanded; falls back to plain text when detail isn't loaded. +const descHtml = computed(() => detail.value?.description_html || null) +// Off-platform file-host links (detail-only), surfaced once expanded. +const externalLinks = computed(() => detail.value?.external_links || []) const descText = computed(() => descExpanded.value ? (fullDescription.value || props.post.description_plain) @@ -219,7 +251,9 @@ onBeforeUnmount(() => { if (ro) { ro.disconnect(); ro = null } }) async function toggleDesc () { if (!descExpanded.value) { - if (props.post.description_truncated && !fullDescription.value && !detail.value) { + // Fetch detail on first expand to get the sanitized HTML body (and the full + // plain text) — render the formatted body, not just the truncated preview. + if (!detail.value) { try { detail.value = await postsStore.getPostFull(props.post.id) } catch { /* render the truncated text rather than nothing */ } @@ -369,6 +403,58 @@ function formatBytes (n) { @container (min-width: 800px) { .fc-post-card__desc--clamped { -webkit-line-clamp: 5; } } +/* Faithful HTML body (expanded). Drop the plain-text pre-wrap and style the + semantic tags the backend sanitizer allows. :deep() pierces scoped CSS to + reach the v-html subtree. */ +.fc-post-card__desc--html { white-space: normal; } +.fc-post-card__desc--html :deep(p) { margin: 0 0 0.6em; } +.fc-post-card__desc--html :deep(p:last-child) { margin-bottom: 0; } +.fc-post-card__desc--html :deep(h1), +.fc-post-card__desc--html :deep(h2), +.fc-post-card__desc--html :deep(h3), +.fc-post-card__desc--html :deep(h4), +.fc-post-card__desc--html :deep(h5), +.fc-post-card__desc--html :deep(h6) { + margin: 0.8em 0 0.4em; + line-height: 1.25; + font-weight: 700; +} +.fc-post-card__desc--html :deep(h1) { font-size: 1.3rem; } +.fc-post-card__desc--html :deep(h2) { font-size: 1.2rem; } +.fc-post-card__desc--html :deep(h3) { font-size: 1.1rem; } +.fc-post-card__desc--html :deep(h4), +.fc-post-card__desc--html :deep(h5), +.fc-post-card__desc--html :deep(h6) { font-size: 1rem; } +.fc-post-card__desc--html :deep(a) { + color: rgb(var(--v-theme-accent)); + text-decoration: underline; + word-break: break-word; +} +.fc-post-card__desc--html :deep(img) { + max-width: 100%; + height: auto; + border-radius: 6px; + margin: 0.4em 0; +} +.fc-post-card__desc--html :deep(ul), +.fc-post-card__desc--html :deep(ol) { margin: 0 0 0.6em; padding-left: 1.4em; } +.fc-post-card__desc--html :deep(li) { margin: 0.15em 0; } +.fc-post-card__desc--html :deep(blockquote) { + margin: 0.6em 0; + padding-left: 0.8em; + border-left: 3px solid rgb(var(--v-theme-on-surface-variant)); + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__desc--html :deep(pre) { + background: rgba(var(--v-theme-on-surface), 0.06); + padding: 0.6em; border-radius: 6px; overflow-x: auto; +} +.fc-post-card__desc--html :deep(code) { font-family: monospace; font-size: 0.85em; } +.fc-post-card__desc--html :deep(hr) { + border: 0; border-top: 1px solid rgb(var(--v-theme-on-surface-variant)); + margin: 0.8em 0; opacity: 0.4; +} +.fc-post-card__desc--html :deep(figure) { margin: 0.4em 0; } .fc-post-card__desc--missing { font-style: italic; color: rgb(var(--v-theme-on-surface-variant)); @@ -404,4 +490,46 @@ function formatBytes (n) { } .fc-post-card__att-icon { color: rgb(var(--v-theme-on-surface-variant)); } .fc-post-card__att-size { color: rgb(var(--v-theme-on-surface-variant)); } + +.fc-post-card__links { + display: flex; flex-direction: column; gap: 6px; + margin-top: 12px; +} +.fc-post-card__links-label { + font-size: 0.75rem; font-weight: 600; text-transform: uppercase; + letter-spacing: 0.04em; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__link { + display: inline-flex; align-items: center; gap: 8px; + padding: 6px 12px; + border: 1px solid rgb(var(--v-theme-on-surface-variant)); + border-radius: 8px; + color: rgb(var(--v-theme-on-surface)); + text-decoration: none; + font-size: 0.85rem; +} +.fc-post-card__link:hover { + color: rgb(var(--v-theme-accent)); + border-color: rgb(var(--v-theme-accent)); +} +.fc-post-card__link-host { + flex: 0 0 auto; + font-weight: 700; text-transform: uppercase; font-size: 0.7rem; + letter-spacing: 0.04em; + padding: 2px 6px; border-radius: 4px; + background: rgba(var(--v-theme-accent), 0.16); + color: rgb(var(--v-theme-accent)); +} +.fc-post-card__link-text { + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} +.fc-post-card__link-status { + flex: 0 0 auto; margin-left: auto; + font-size: 0.7rem; text-transform: uppercase; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__link-status[data-status="downloaded"] { + color: rgb(var(--v-theme-accent)); +} diff --git a/frontend/src/components/subscriptions/SettingsTab.vue b/frontend/src/components/subscriptions/SettingsTab.vue index 5b6c3b2..3d86e13 100644 --- a/frontend/src/components/subscriptions/SettingsTab.vue +++ b/frontend/src/components/subscriptions/SettingsTab.vue @@ -52,6 +52,52 @@ +

External file-host downloads

+ + +
+ When a creator links files on these hosts in a post, Curator downloads + them into the post automatically (tagged + provenance-linked like any + other media). Turn a host off if it starts rate-limiting or banning. +
+ + + + + + + + + + + + + + + + + +
+ + + +
+

Schedule defaults

@@ -151,6 +197,11 @@ const dl = reactive({ download_schedule_default_seconds: 28800, download_event_retention_days: 90, download_failure_warning_threshold: 5, + extdl_mega_enabled: true, + extdl_gdrive_enabled: true, + extdl_mediafire_enabled: true, + extdl_dropbox_enabled: true, + extdl_pixeldrain_enabled: true, }) watch(() => importStore.settings, (s) => { if (s) Object.assign(dl, s) }, { immediate: true }) diff --git a/frontend/src/views/SeriesView.vue b/frontend/src/views/SeriesView.vue index cc847ef..552883a 100644 --- a/frontend/src/views/SeriesView.vue +++ b/frontend/src/views/SeriesView.vue @@ -1,35 +1,60 @@