Wide-window feed, Discord downloads named and dated correctly, and a repair for the broken ones #253
@@ -475,6 +475,20 @@ async def trigger_reclaim_attachments():
|
|||||||
return _queued(async_result)
|
return _queued(async_result)
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
|
||||||
|
async def trigger_repair_discord_downloads():
|
||||||
|
"""Clean re-download of the Discord files broken by the `None` naming
|
||||||
|
(#3999). Body {"dry_run": bool}; dry_run is the DEFAULT, because the apply
|
||||||
|
deletes files and makes gallery-dl forget every Discord download. Returns the
|
||||||
|
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||||
|
from ..tasks.admin import repair_discord_downloads_task
|
||||||
|
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
dry_run = bool(body.get("dry_run", True))
|
||||||
|
async_result = repair_discord_downloads_task.delay(dry_run=dry_run)
|
||||||
|
return _queued(async_result)
|
||||||
|
|
||||||
|
|
||||||
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
||||||
async def trigger_dedup_videos():
|
async def trigger_dedup_videos():
|
||||||
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Repair the Discord downloads made before the naming fix (issue #3999).
|
||||||
|
|
||||||
|
Until dc840fe, gallery-dl's Discord patterns asked for keys the extractor never
|
||||||
|
emits, so every Discord download landed as
|
||||||
|
`<artist>/discord/None/<date>_None_<original name>`, next to a sidecar named
|
||||||
|
after the attachment's ORIGINAL name. That broke two things:
|
||||||
|
|
||||||
|
* **No Post, no date.** `find_sidecar` can never pair those names, so these
|
||||||
|
files were imported as loose images with no Post, and a card shows the
|
||||||
|
download time.
|
||||||
|
* **No trustworthy metadata to relink from.** Every `image.png` in a channel
|
||||||
|
wrote the same `image.json`, so the surviving sidecar describes whichever
|
||||||
|
message was written last. The message id is gone from the filename too.
|
||||||
|
|
||||||
|
The operator chose a clean re-download (2026-09-13) over relinking in place:
|
||||||
|
delete the broken files and their records, make gallery-dl forget it fetched
|
||||||
|
them, and backfill every Discord source again under the fixed naming.
|
||||||
|
|
||||||
|
## Why the archive is cleared for ALL of Discord
|
||||||
|
|
||||||
|
gallery-dl records a download as `discord{message_id}_{num}` (upstream
|
||||||
|
`DiscordExtractor.archive_fmt`, prefixed with the category). The broken files
|
||||||
|
lost their message ids, so there is no way to forget one source's entries and
|
||||||
|
not another's. Every Discord download made before the fix is broken, so
|
||||||
|
forgetting all of them is exactly right. Anything downloaded AFTER the fix still
|
||||||
|
exists on disk under its correct name, and gallery-dl's `skip` sees the file and
|
||||||
|
does not fetch it again.
|
||||||
|
|
||||||
|
## What it does not touch
|
||||||
|
|
||||||
|
Discord posts FC grouped itself (#388 E2) are built from Posts, and these files
|
||||||
|
never had one, so there is nothing grouped to unwind. Images outside a
|
||||||
|
`discord/None/` directory are never selected: the path filter requires both the
|
||||||
|
`None` directory and the `_None_` filename, the pair only the bug produced.
|
||||||
|
|
||||||
|
Operator-triggered only (Settings, preview first). Never on a beat.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import ImageRecord, Source
|
||||||
|
from .cleanup_service import delete_images
|
||||||
|
from .gallery_dl import archive_path
|
||||||
|
from .source_service import arm_backfill
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# `%` and `_` are LIKE wildcards, so the literal underscores around None are
|
||||||
|
# escaped. `________` is the eight-digit date prefix the old pattern wrote.
|
||||||
|
_BROKEN_PATH_LIKE = r"%/discord/None/________\_None\_%"
|
||||||
|
|
||||||
|
# Upstream keys asset downloads as `asset_{server_id}_{id}`. FC never fetches
|
||||||
|
# server assets, but excluding them keeps this to exactly the message
|
||||||
|
# attachments the bug mangled.
|
||||||
|
_ARCHIVE_SQL_MATCH = r"entry LIKE 'discord%' AND entry NOT LIKE 'discordasset\_%' ESCAPE '\'"
|
||||||
|
_COUNT_SQL = "SELECT COUNT(*) FROM archive WHERE " + _ARCHIVE_SQL_MATCH
|
||||||
|
_DELETE_SQL = "DELETE FROM archive WHERE " + _ARCHIVE_SQL_MATCH
|
||||||
|
|
||||||
|
|
||||||
|
def broken_directories(images_root: Path) -> list[Path]:
|
||||||
|
"""Every `<artist>/discord/None` directory. Artist folders that differ only by
|
||||||
|
case (`Conto` and `conto`) are separate directories and both are found."""
|
||||||
|
return sorted(d for d in Path(images_root).glob("*/discord/None") if d.is_dir())
|
||||||
|
|
||||||
|
|
||||||
|
def count_archive_entries(archive: Path) -> int:
|
||||||
|
return _archive(archive, delete=False)
|
||||||
|
|
||||||
|
|
||||||
|
def forget_archive_entries(archive: Path) -> int:
|
||||||
|
return _archive(archive, delete=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _archive(archive: Path, *, delete: bool) -> int:
|
||||||
|
if not archive.is_file():
|
||||||
|
return 0
|
||||||
|
# A download running at the same moment holds this file briefly. Waiting
|
||||||
|
# 30s for its lock beats failing the repair over a transient contention.
|
||||||
|
conn = sqlite3.connect(str(archive), timeout=30)
|
||||||
|
try:
|
||||||
|
has_table = conn.execute(
|
||||||
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='archive'"
|
||||||
|
).fetchone()
|
||||||
|
if not has_table:
|
||||||
|
return 0
|
||||||
|
if not delete:
|
||||||
|
return conn.execute(_COUNT_SQL).fetchone()[0]
|
||||||
|
cur = conn.execute(_DELETE_SQL)
|
||||||
|
conn.commit()
|
||||||
|
return cur.rowcount
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _sweep_directory(directory: Path) -> tuple[int, bool]:
|
||||||
|
"""Remove what the record deletes left behind: the collided sidecars, plus any
|
||||||
|
file that never became a record (a quarantined or rejected download). Then
|
||||||
|
the directory itself, if it is empty. Returns (files removed, dir removed)."""
|
||||||
|
removed = 0
|
||||||
|
for f in directory.iterdir():
|
||||||
|
if f.is_file():
|
||||||
|
try:
|
||||||
|
f.unlink()
|
||||||
|
removed += 1
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning("discord repair: could not remove %s: %s", f, exc)
|
||||||
|
try:
|
||||||
|
directory.rmdir()
|
||||||
|
return removed, True
|
||||||
|
except OSError:
|
||||||
|
return removed, False
|
||||||
|
|
||||||
|
|
||||||
|
def repair_discord_downloads(
|
||||||
|
session: Session, *, images_root: Path, dry_run: bool,
|
||||||
|
) -> dict:
|
||||||
|
images_root = Path(images_root)
|
||||||
|
archive = archive_path(images_root)
|
||||||
|
|
||||||
|
broken = select(ImageRecord.id, ImageRecord.size_bytes).where(
|
||||||
|
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
|
||||||
|
)
|
||||||
|
rows = session.execute(broken).all()
|
||||||
|
image_ids = [r.id for r in rows]
|
||||||
|
directories = broken_directories(images_root)
|
||||||
|
sources = session.execute(
|
||||||
|
select(Source).where(Source.platform == "discord")
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"images": len(image_ids),
|
||||||
|
"bytes": sum(r.size_bytes or 0 for r in rows),
|
||||||
|
"directories": len(directories),
|
||||||
|
"sources": len(sources),
|
||||||
|
"enabled_sources": sum(1 for s in sources if s.enabled),
|
||||||
|
}
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
summary["archive_entries"] = count_archive_entries(archive)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
deleted = delete_images(session, image_ids=image_ids, images_root=images_root)
|
||||||
|
|
||||||
|
swept = 0
|
||||||
|
directories_removed = 0
|
||||||
|
for d in directories:
|
||||||
|
n, gone = _sweep_directory(d)
|
||||||
|
swept += n
|
||||||
|
directories_removed += int(gone)
|
||||||
|
|
||||||
|
# Only after the files are gone. Forgetting first and failing half way would
|
||||||
|
# leave gallery-dl free to re-fetch into a directory still full of the old
|
||||||
|
# copies.
|
||||||
|
forgotten = forget_archive_entries(archive)
|
||||||
|
|
||||||
|
for source in sources:
|
||||||
|
arm_backfill(source)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
remaining = session.execute(
|
||||||
|
select(func.count(ImageRecord.id)).where(
|
||||||
|
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
summary.update(
|
||||||
|
images_deleted=deleted["images_deleted"],
|
||||||
|
files_failed=deleted["files_failed"],
|
||||||
|
leftover_files_removed=swept,
|
||||||
|
directories_removed=directories_removed,
|
||||||
|
archive_entries=forgotten,
|
||||||
|
backfills_started=len(sources),
|
||||||
|
remaining=remaining,
|
||||||
|
)
|
||||||
|
log.info("discord repair applied: %s", summary)
|
||||||
|
return summary
|
||||||
@@ -94,6 +94,58 @@ BACKFILL_CHUNK_SECONDS = 600
|
|||||||
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||||
|
|
||||||
|
|
||||||
|
# --- Discord naming ---------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
|
||||||
|
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
|
||||||
|
# for an attachment: `channel` is a plain STRING (the channel's name), the
|
||||||
|
# message is `message_id`, the attachment's position in it is `num`, and there
|
||||||
|
# is NO `id` key at all.
|
||||||
|
#
|
||||||
|
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
|
||||||
|
# "None", so every Discord download since the platform was added landed in a
|
||||||
|
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
|
||||||
|
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
|
||||||
|
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
|
||||||
|
# got a Post or a post date, and (b) collides: every `image.png` in a channel
|
||||||
|
# overwrote the same `image.json`, so the one sidecar that survived described
|
||||||
|
# whichever message happened to be written last.
|
||||||
|
#
|
||||||
|
# The fix names the sidecar EXACTLY like the media minus its extension, so
|
||||||
|
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
|
||||||
|
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
|
||||||
|
# these patterns against a sanitized copy of the real sidecar, so a key that
|
||||||
|
# does not exist fails CI instead of silently becoming "None".
|
||||||
|
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
|
||||||
|
DISCORD_DIRECTORY = ["{channel}"]
|
||||||
|
|
||||||
|
|
||||||
|
def sidecar_name_for(media_pattern: str) -> str | None:
|
||||||
|
"""The metadata filename pattern that names a sidecar exactly like its media.
|
||||||
|
|
||||||
|
Returns None for a pattern that does not end in `.{extension}`, since then
|
||||||
|
there is no media stem to mirror and the caller must fall back.
|
||||||
|
"""
|
||||||
|
suffix = ".{extension}"
|
||||||
|
if not media_pattern.endswith(suffix):
|
||||||
|
return None
|
||||||
|
return media_pattern[: -len(suffix)] + ".json"
|
||||||
|
|
||||||
|
|
||||||
|
def metadata_postprocessor(filename: str) -> dict:
|
||||||
|
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
|
||||||
|
|
||||||
|
|
||||||
|
def archive_path(images_root: Path) -> Path:
|
||||||
|
"""gallery-dl's download archive: the record of what it has already fetched.
|
||||||
|
|
||||||
|
One definition, because the Discord repair (services/discord_repair.py) has
|
||||||
|
to find the same file the downloader writes, without constructing a service
|
||||||
|
whose __init__ creates directories.
|
||||||
|
"""
|
||||||
|
return Path(images_root) / ".gallery-dl" / "archive.sqlite3"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SourceConfig:
|
class SourceConfig:
|
||||||
"""Per-source overrides loaded from Source.config_overrides JSON.
|
"""Per-source overrides loaded from Source.config_overrides JSON.
|
||||||
@@ -376,8 +428,11 @@ class GalleryDLService:
|
|||||||
},
|
},
|
||||||
"discord": {
|
"discord": {
|
||||||
"content_types": ["all"],
|
"content_types": ["all"],
|
||||||
"directory": ["{channel[name]}"],
|
"directory": DISCORD_DIRECTORY,
|
||||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
"filename": DISCORD_FILENAME,
|
||||||
|
# Overrides the global `{filename}.json` sidecar for this extractor
|
||||||
|
# only — see the Discord naming note above.
|
||||||
|
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
|
||||||
"embeds": "all",
|
"embeds": "all",
|
||||||
"stickers": True,
|
"stickers": True,
|
||||||
"reactions": False,
|
"reactions": False,
|
||||||
@@ -402,7 +457,7 @@ class GalleryDLService:
|
|||||||
config = {
|
config = {
|
||||||
"extractor": {
|
"extractor": {
|
||||||
"base-directory": str(self.images_root),
|
"base-directory": str(self.images_root),
|
||||||
"archive": str(self._config_dir / "archive.sqlite3"),
|
"archive": str(archive_path(self.images_root)),
|
||||||
"skip": True,
|
"skip": True,
|
||||||
"sleep": self._rate_limit,
|
"sleep": self._rate_limit,
|
||||||
"sleep-request": max(0.5, self._rate_limit / 4),
|
"sleep-request": max(0.5, self._rate_limit / 4),
|
||||||
@@ -497,6 +552,17 @@ class GalleryDLService:
|
|||||||
if source_config.filename_pattern:
|
if source_config.filename_pattern:
|
||||||
platform_section["filename"] = source_config.filename_pattern
|
platform_section["filename"] = source_config.filename_pattern
|
||||||
|
|
||||||
|
# A platform that names its sidecar after its media must keep doing so
|
||||||
|
# under a per-source filename override, or the pairing breaks exactly the
|
||||||
|
# way Discord's did. No metadata wanted means no platform postprocessor
|
||||||
|
# either — the global list was already dropped above.
|
||||||
|
if "postprocessors" in platform_section:
|
||||||
|
mirrored = sidecar_name_for(platform_section.get("filename") or "")
|
||||||
|
if not source_config.save_metadata or mirrored is None:
|
||||||
|
platform_section.pop("postprocessors")
|
||||||
|
else:
|
||||||
|
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
|
||||||
|
|
||||||
platform_section["metadata"] = source_config.save_metadata
|
platform_section["metadata"] = source_config.save_metadata
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|||||||
@@ -158,6 +158,19 @@ def _is_app_managed(key: str) -> bool:
|
|||||||
BACKFILL_MAX_CHUNKS = 200
|
BACKFILL_MAX_CHUNKS = 200
|
||||||
|
|
||||||
|
|
||||||
|
def arm_backfill(source: Source) -> None:
|
||||||
|
"""Arm a fresh run-until-done backfill on `source` (plan #693). Mutation
|
||||||
|
only — the caller commits. Shared by `SourceService.start_backfill` and the
|
||||||
|
sync Discord repair, so both clear exactly the same resume state."""
|
||||||
|
co = dict(source.config_overrides or {})
|
||||||
|
co["_backfill_state"] = "running"
|
||||||
|
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||||
|
"_backfill_posts"):
|
||||||
|
co.pop(k, None)
|
||||||
|
source.config_overrides = co
|
||||||
|
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||||
|
|
||||||
|
|
||||||
class SourceService:
|
class SourceService:
|
||||||
def __init__(self, session: AsyncSession):
|
def __init__(self, session: AsyncSession):
|
||||||
self.session = session
|
self.session = session
|
||||||
@@ -473,13 +486,7 @@ class SourceService:
|
|||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
if source is None:
|
if source is None:
|
||||||
raise LookupError(f"source id={source_id} not found")
|
raise LookupError(f"source id={source_id} not found")
|
||||||
co = dict(source.config_overrides or {})
|
arm_backfill(source)
|
||||||
co["_backfill_state"] = "running"
|
|
||||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
|
||||||
"_backfill_posts"):
|
|
||||||
co.pop(k, None)
|
|
||||||
source.config_overrides = co
|
|
||||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
|
||||||
await self.session.commit()
|
await self.session.commit()
|
||||||
return await self._row_to_record(source)
|
return await self._row_to_record(source)
|
||||||
|
|
||||||
|
|||||||
@@ -122,6 +122,28 @@ def prune_missing_file_records_task(self) -> dict:
|
|||||||
return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
|
return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.repair_discord_downloads_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||||
|
)
|
||||||
|
def repair_discord_downloads_task(self, dry_run: bool = True) -> dict:
|
||||||
|
"""Clean re-download of the Discord files broken by the `None` naming
|
||||||
|
(#3999). dry_run (the default) returns the projection; apply deletes the
|
||||||
|
broken images and their files, makes gallery-dl forget every Discord
|
||||||
|
download, and restarts every Discord source's backfill. Defaults to the SAFE
|
||||||
|
preview because the apply deletes files. Operator-triggered only."""
|
||||||
|
from ..services.discord_repair import repair_discord_downloads
|
||||||
|
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
return repair_discord_downloads(
|
||||||
|
session, images_root=IMAGES_ROOT, dry_run=dry_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.admin.dedup_videos_task",
|
name="backend.app.tasks.admin.dedup_videos_task",
|
||||||
bind=True,
|
bind=True,
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
<template>
|
<template>
|
||||||
<v-card class="fc-post-card" variant="outlined">
|
<v-card
|
||||||
|
ref="cardEl" class="fc-post-card" variant="outlined"
|
||||||
|
:class="{ 'fc-post-card--wide': wide }"
|
||||||
|
>
|
||||||
<div class="fc-post-card__head">
|
<div class="fc-post-card__head">
|
||||||
<!-- Posts with no live subscription have source=null (alembic 0030);
|
<!-- Posts with no live subscription have source=null (alembic 0030);
|
||||||
show a "filesystem import" affordance instead of a platform chip. -->
|
show a "filesystem import" affordance instead of a platform chip. -->
|
||||||
@@ -58,7 +61,7 @@
|
|||||||
</button>
|
</button>
|
||||||
<div
|
<div
|
||||||
v-if="rail.length || moreCount" class="fc-post-card__rail"
|
v-if="rail.length || moreCount" class="fc-post-card__rail"
|
||||||
:style="{ '--fc-rail-cols': railCols }"
|
:style="{ '--fc-rail-cols': railCols, '--fc-grid-cols': Math.min(2, railCols) }"
|
||||||
>
|
>
|
||||||
<button
|
<button
|
||||||
v-for="t in rail" :key="t.image_id" type="button"
|
v-for="t in rail" :key="t.image_id" type="button"
|
||||||
@@ -221,20 +224,33 @@ const synthesisTitle = computed(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const hero = computed(() => images.value[0])
|
const hero = computed(() => images.value[0])
|
||||||
// The thumbnail strip spans the hero's full width (CSS grid, equal columns),
|
|
||||||
// rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are
|
// Filmstrip layout (milestone #407, option A). On a very wide window a card
|
||||||
// more images than fit, the last cell becomes a "+N" overflow tile so the
|
// that just grew would give one post the whole screen, so a WIDE card instead
|
||||||
// count stays accurate.
|
// pins the hero to a fixed height and moves the extra images into a 2-column
|
||||||
const RAIL_MAX = 5
|
// grid BESIDE it. Measured on the card rather than the viewport because the
|
||||||
|
// same card renders in the Latest feed, Browse, and the in-context view, each
|
||||||
|
// at a different width.
|
||||||
|
const WIDE_CARD_PX = 1100
|
||||||
|
const cardEl = ref(null)
|
||||||
|
const wide = ref(false)
|
||||||
|
|
||||||
|
// The narrow layout's strip spans the hero's full width (CSS grid, equal
|
||||||
|
// columns); the wide layout's grid is 2×2. Show up to that many cells; when
|
||||||
|
// there are more images than fit, the last cell becomes a "+N" overflow tile so
|
||||||
|
// the count stays accurate.
|
||||||
|
const RAIL_MAX_NARROW = 5
|
||||||
|
const RAIL_MAX_WIDE = 4
|
||||||
const serverMore = computed(() => props.post.thumbnails_more || 0)
|
const serverMore = computed(() => props.post.thumbnails_more || 0)
|
||||||
const afterHero = computed(() => images.value.slice(1))
|
const afterHero = computed(() => images.value.slice(1))
|
||||||
|
const railMax = computed(() => (wide.value ? RAIL_MAX_WIDE : RAIL_MAX_NARROW))
|
||||||
const hasOverflow = computed(
|
const hasOverflow = computed(
|
||||||
() => serverMore.value > 0 || afterHero.value.length > RAIL_MAX,
|
() => serverMore.value > 0 || afterHero.value.length > railMax.value,
|
||||||
)
|
)
|
||||||
const rail = computed(() =>
|
const rail = computed(() =>
|
||||||
hasOverflow.value
|
hasOverflow.value
|
||||||
? afterHero.value.slice(0, RAIL_MAX - 1)
|
? afterHero.value.slice(0, railMax.value - 1)
|
||||||
: afterHero.value.slice(0, RAIL_MAX),
|
: afterHero.value.slice(0, railMax.value),
|
||||||
)
|
)
|
||||||
const visibleCount = computed(() => (images.value.length ? 1 + rail.value.length : 0))
|
const visibleCount = computed(() => (images.value.length ? 1 + rail.value.length : 0))
|
||||||
const moreCount = computed(() => {
|
const moreCount = computed(() => {
|
||||||
@@ -349,8 +365,17 @@ function measureOverflow () {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let ro = null
|
let ro = null
|
||||||
|
let cardRo = null
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
nextTick(measureOverflow)
|
nextTick(measureOverflow)
|
||||||
|
const root = cardEl.value?.$el
|
||||||
|
if (typeof ResizeObserver !== 'undefined' && root) {
|
||||||
|
cardRo = new ResizeObserver((entries) => {
|
||||||
|
const w = entries[0]?.contentRect?.width ?? 0
|
||||||
|
wide.value = w >= WIDE_CARD_PX
|
||||||
|
})
|
||||||
|
cardRo.observe(root)
|
||||||
|
}
|
||||||
// Re-measure when the card resizes (the container-query clamp differs by
|
// Re-measure when the card resizes (the container-query clamp differs by
|
||||||
// width). Guarded for happy-dom / older runtimes without ResizeObserver.
|
// width). Guarded for happy-dom / older runtimes without ResizeObserver.
|
||||||
if (typeof ResizeObserver !== 'undefined' && descEl.value) {
|
if (typeof ResizeObserver !== 'undefined' && descEl.value) {
|
||||||
@@ -358,7 +383,10 @@ onMounted(() => {
|
|||||||
ro.observe(descEl.value)
|
ro.observe(descEl.value)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
onBeforeUnmount(() => { if (ro) { ro.disconnect(); ro = null } })
|
onBeforeUnmount(() => {
|
||||||
|
if (ro) { ro.disconnect(); ro = null }
|
||||||
|
if (cardRo) { cardRo.disconnect(); cardRo = null }
|
||||||
|
})
|
||||||
|
|
||||||
async function toggleDesc () {
|
async function toggleDesc () {
|
||||||
if (!descExpanded.value) {
|
if (!descExpanded.value) {
|
||||||
@@ -495,6 +523,40 @@ function formatBytes (n) {
|
|||||||
color: rgb(var(--v-theme-accent));
|
color: rgb(var(--v-theme-accent));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --- Filmstrip layout (wide cards, #407 A) ---------------------------------
|
||||||
|
The hero has a HEIGHT, not a width: a wide card must not turn into a
|
||||||
|
full-screen post, so its height stays roughly a third of the viewport
|
||||||
|
whatever the window's width. The extra images sit beside it as square cells
|
||||||
|
whose size derives from that same height, so the grid always ends flush with
|
||||||
|
the hero's bottom edge. */
|
||||||
|
.fc-post-card--wide {
|
||||||
|
--fc-hero-h: clamp(260px, 34vh, 460px);
|
||||||
|
--fc-grid-gap: 8px;
|
||||||
|
--fc-cell: calc((var(--fc-hero-h) - var(--fc-grid-gap)) / 2);
|
||||||
|
}
|
||||||
|
.fc-post-card--wide .fc-post-card__body { flex-direction: row; gap: 24px; }
|
||||||
|
.fc-post-card--wide .fc-post-card__media {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
display: flex;
|
||||||
|
gap: var(--fc-grid-gap);
|
||||||
|
}
|
||||||
|
.fc-post-card--wide .fc-post-card__hero {
|
||||||
|
width: auto;
|
||||||
|
height: var(--fc-hero-h);
|
||||||
|
}
|
||||||
|
.fc-post-card--wide .fc-post-card__rail {
|
||||||
|
margin-top: 0;
|
||||||
|
gap: var(--fc-grid-gap);
|
||||||
|
grid-template-columns: repeat(var(--fc-grid-cols, 2), var(--fc-cell));
|
||||||
|
grid-auto-rows: var(--fc-cell);
|
||||||
|
}
|
||||||
|
.fc-post-card--wide .fc-post-card__rail-cell,
|
||||||
|
.fc-post-card--wide .fc-post-card__rail-more { height: 100%; }
|
||||||
|
.fc-post-card--wide .fc-post-card__text { flex: 1 1 0; min-width: 0; }
|
||||||
|
/* Text is secondary here — long reads happen in the expanded view — so the
|
||||||
|
clamp keeps the text column no taller than the images beside it. */
|
||||||
|
.fc-post-card--wide .fc-post-card__desc--clamped { -webkit-line-clamp: 4; }
|
||||||
|
|
||||||
.fc-post-card__title {
|
.fc-post-card__title {
|
||||||
font-family: 'Fraunces', Georgia, serif;
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
font-size: 18px; font-weight: 700;
|
font-size: 18px; font-weight: 700;
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
<template>
|
||||||
|
<!-- #3999: Discord downloads made before the naming fix landed in a `None`
|
||||||
|
folder with no post and a download-time date. This re-downloads them
|
||||||
|
cleanly. Preview first; the apply deletes files. -->
|
||||||
|
<MaintenanceTile
|
||||||
|
icon="mdi-download-off-outline"
|
||||||
|
title="Repair Discord downloads"
|
||||||
|
blurb="Re-download Discord images that were saved with no post and the wrong date."
|
||||||
|
destructive
|
||||||
|
:open="applying || previewing"
|
||||||
|
>
|
||||||
|
<p class="text-body-2 mb-3">
|
||||||
|
Before the naming fix, every Discord download was saved into a folder
|
||||||
|
called <code>None</code>. Those images never got a post, so they show the
|
||||||
|
time Curator downloaded them rather than when they were posted.
|
||||||
|
<strong>Apply</strong> deletes those images, clears Discord from the
|
||||||
|
download history, and starts a fresh backfill of every Discord source so
|
||||||
|
they come back with their posts and dates. Tags you added by hand to those
|
||||||
|
images are lost. Nothing outside a Discord <code>None</code> folder is
|
||||||
|
touched.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||||
|
<v-btn
|
||||||
|
color="primary" variant="tonal" rounded="pill"
|
||||||
|
:loading="previewing" :disabled="applying" @click="preview"
|
||||||
|
>
|
||||||
|
<v-icon start>mdi-magnify</v-icon> Preview
|
||||||
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
color="error" rounded="pill"
|
||||||
|
:loading="applying"
|
||||||
|
:disabled="previewing || !canApply"
|
||||||
|
@click="confirmOpen = true"
|
||||||
|
>
|
||||||
|
<v-icon start>mdi-download-off-outline</v-icon> Apply
|
||||||
|
</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||||
|
density="comfortable"
|
||||||
|
>
|
||||||
|
<span v-if="applied">
|
||||||
|
Deleted {{ summary.images_deleted }} image(s) and cleared
|
||||||
|
{{ summary.archive_entries }} Discord download record(s). Backfills
|
||||||
|
started on {{ summary.backfills_started }} Discord source(s).
|
||||||
|
</span>
|
||||||
|
<span v-else-if="hasWork">
|
||||||
|
{{ summary.images }} broken image(s) ({{ humanBytes(summary.bytes) }})
|
||||||
|
across {{ summary.directories }} folder(s), and
|
||||||
|
{{ summary.archive_entries }} Discord download record(s) to clear.
|
||||||
|
{{ summary.sources }} Discord source(s) will backfill again.
|
||||||
|
</span>
|
||||||
|
<span v-else>Nothing to repair — no broken Discord downloads found.</span>
|
||||||
|
|
||||||
|
<div v-if="applied && summary.files_failed" class="mt-1 text-caption">
|
||||||
|
{{ summary.files_failed }} file(s) could not be removed — see the worker log.
|
||||||
|
</div>
|
||||||
|
<div v-if="applied && summary.remaining" class="mt-1 text-caption">
|
||||||
|
{{ summary.remaining }} broken image(s) are still present. Run it again.
|
||||||
|
</div>
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<QueueStatusBar queue="maintenance_long" queue-label="Maintenance" />
|
||||||
|
|
||||||
|
<v-dialog v-model="confirmOpen" max-width="440">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Repair Discord downloads?</v-card-title>
|
||||||
|
<v-card-text class="text-body-2">
|
||||||
|
This permanently deletes <strong>{{ summary?.images ?? 0 }}</strong>
|
||||||
|
Discord image(s) ({{ humanBytes(summary?.bytes) }}) and re-downloads
|
||||||
|
them from Discord with their posts and dates. Any tags you added to
|
||||||
|
those images by hand will not come back.
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||||
|
<v-btn color="error" @click="apply">Repair</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</MaintenanceTile>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||||
|
import { humanBytes } from '../../utils/bytes.js'
|
||||||
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
|
|
||||||
|
const confirmOpen = ref(false)
|
||||||
|
|
||||||
|
const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
|
||||||
|
endpoint: '/api/admin/maintenance/repair-discord-downloads',
|
||||||
|
storageKey: 'fc.maint.repairDiscordDownloads',
|
||||||
|
appliedToast: 'Discord downloads cleared — backfills started',
|
||||||
|
})
|
||||||
|
|
||||||
|
// The archive count matters too: files could already be gone while gallery-dl
|
||||||
|
// still believes it has them, which would stop the backfill re-fetching.
|
||||||
|
const hasWork = computed(
|
||||||
|
() => !!summary.value && (summary.value.images > 0 || summary.value.archive_entries > 0),
|
||||||
|
)
|
||||||
|
const canApply = computed(() => hasWork.value && !applied.value)
|
||||||
|
const summaryType = computed(() => {
|
||||||
|
if (applied.value) return 'success'
|
||||||
|
return hasWork.value ? 'info' : 'success'
|
||||||
|
})
|
||||||
|
|
||||||
|
// The confirm dialog gates the destructive apply; close it, then run.
|
||||||
|
function apply () {
|
||||||
|
confirmOpen.value = false
|
||||||
|
applyTask()
|
||||||
|
}
|
||||||
|
</script>
|
||||||
@@ -15,6 +15,7 @@
|
|||||||
<ImportFiltersForm />
|
<ImportFiltersForm />
|
||||||
<TranslationCard />
|
<TranslationCard />
|
||||||
<DiscordGroupingCard />
|
<DiscordGroupingCard />
|
||||||
|
<DiscordRepairCard />
|
||||||
<PostAssociationsCard />
|
<PostAssociationsCard />
|
||||||
<MembershipRosterCard />
|
<MembershipRosterCard />
|
||||||
<MembershipSuggestionsCard />
|
<MembershipSuggestionsCard />
|
||||||
@@ -85,6 +86,7 @@ import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
|||||||
import CropProposersCard from './CropProposersCard.vue'
|
import CropProposersCard from './CropProposersCard.vue'
|
||||||
import HeadsCard from './HeadsCard.vue'
|
import HeadsCard from './HeadsCard.vue'
|
||||||
import DiscordGroupingCard from './DiscordGroupingCard.vue'
|
import DiscordGroupingCard from './DiscordGroupingCard.vue'
|
||||||
|
import DiscordRepairCard from './DiscordRepairCard.vue'
|
||||||
import MembershipRosterCard from './MembershipRosterCard.vue'
|
import MembershipRosterCard from './MembershipRosterCard.vue'
|
||||||
import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue'
|
import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue'
|
||||||
import PostAssociationsCard from './PostAssociationsCard.vue'
|
import PostAssociationsCard from './PostAssociationsCard.vue'
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
<template>
|
<template>
|
||||||
<v-container class="pt-2 pb-6" max-width="900">
|
<!-- Width is set in CSS, not with `max-width` here: below 1600px it is
|
||||||
|
today's 900px column, and above it the feed widens and gains the rail
|
||||||
|
and day gutter (milestone #407). -->
|
||||||
|
<v-container fluid class="pt-2 pb-6 fc-posts">
|
||||||
<!-- In-context view: deep-linked to one post, with bidirectional infinite
|
<!-- In-context view: deep-linked to one post, with bidirectional infinite
|
||||||
scroll — newer posts load above, older posts below. -->
|
scroll — newer posts load above, older posts below. -->
|
||||||
<template v-if="postIdFilter != null">
|
<template v-if="postIdFilter != null">
|
||||||
@@ -48,15 +51,19 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<!-- Normal feed -->
|
<!-- Normal feed -->
|
||||||
<template v-else>
|
<div v-else class="fc-posts__layout">
|
||||||
<FeedStatusRibbon v-if="statusRibbon" />
|
<!-- On a wide window this is a sticky left rail (#407 E); below the
|
||||||
|
breakpoint it lays out exactly as the old inline header did. -->
|
||||||
|
<aside class="fc-posts__rail">
|
||||||
<PostsFilterBar
|
<PostsFilterBar
|
||||||
:artist-id="artistFilter"
|
:artist-id="artistFilter"
|
||||||
:platform="platformFilter"
|
:platform="platformFilter"
|
||||||
@update:filters="onFilters"
|
@update:filters="onFilters"
|
||||||
/>
|
/>
|
||||||
|
<FeedStatusRibbon v-if="statusRibbon" />
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<div class="fc-posts__main">
|
||||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||||
{{ String(store.error) }}
|
{{ String(store.error) }}
|
||||||
</v-alert>
|
</v-alert>
|
||||||
@@ -74,14 +81,28 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else>
|
<div v-else>
|
||||||
<PostCard v-for="p in store.items" :key="p.id" :post="p" />
|
<!-- Day groups (#407 D). The heading sits above its posts on a narrow
|
||||||
|
window and in a sticky left gutter on a wide one. -->
|
||||||
|
<section v-for="d in days" :key="d.key" class="fc-posts__day">
|
||||||
|
<header class="fc-posts__day-head">
|
||||||
|
<span class="fc-posts__day-label">{{ d.label }}</span>
|
||||||
|
<span class="fc-posts__day-count">
|
||||||
|
{{ d.posts.length }} post{{ d.posts.length === 1 ? '' : 's' }}
|
||||||
|
· {{ d.artistCount }} artist{{ d.artistCount === 1 ? '' : 's' }}
|
||||||
|
</span>
|
||||||
|
</header>
|
||||||
|
<div class="fc-posts__day-posts">
|
||||||
|
<PostCard v-for="p in d.posts" :key="p.id" :post="p" />
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
<div ref="sentinel" class="fc-posts__sentinel">
|
<div ref="sentinel" class="fc-posts__sentinel">
|
||||||
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
|
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
|
||||||
<span v-else-if="store.done" class="fc-posts__end">End of stream</span>
|
<span v-else-if="store.done" class="fc-posts__end">End of stream</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</div>
|
||||||
|
</div>
|
||||||
</v-container>
|
</v-container>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -121,6 +142,48 @@ const hasActiveFilter = computed(() =>
|
|||||||
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
|
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// --- day groups (#407 D) ---
|
||||||
|
// CONSECUTIVE runs, not a bucket per date. The feed's sort key includes
|
||||||
|
// `resurfaced_at` (a Discord grouping that grew moves back to the top), which
|
||||||
|
// the payload does not carry, so a resurfaced post can sit above newer ones.
|
||||||
|
// Bucketing by date would pull it out of order; a run gives it its own heading
|
||||||
|
// where it actually appears. Counts cover what has LOADED, and grow as the
|
||||||
|
// infinite scroll fetches more of the same day.
|
||||||
|
function dayKey (iso) {
|
||||||
|
const d = new Date(iso)
|
||||||
|
return `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`
|
||||||
|
}
|
||||||
|
function dayLabel (iso) {
|
||||||
|
const d = new Date(iso)
|
||||||
|
const today = new Date()
|
||||||
|
const startOf = (x) => new Date(x.getFullYear(), x.getMonth(), x.getDate()).getTime()
|
||||||
|
const days = Math.round((startOf(today) - startOf(d)) / 86400000)
|
||||||
|
if (days === 0) return 'Today'
|
||||||
|
if (days === 1) return 'Yesterday'
|
||||||
|
if (days > 1 && days < 7) return d.toLocaleDateString(undefined, { weekday: 'long' })
|
||||||
|
const sameYear = d.getFullYear() === today.getFullYear()
|
||||||
|
return d.toLocaleDateString(undefined, {
|
||||||
|
month: 'short', day: 'numeric', ...(sameYear ? {} : { year: 'numeric' }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const days = computed(() => {
|
||||||
|
const groups = []
|
||||||
|
for (const p of store.items) {
|
||||||
|
const iso = p.post_date || p.downloaded_at
|
||||||
|
const key = dayKey(iso)
|
||||||
|
let g = groups[groups.length - 1]
|
||||||
|
if (!g || g.dayKey !== key) {
|
||||||
|
// Suffix with the run index so a day that appears twice (see above)
|
||||||
|
// still has a unique v-for key.
|
||||||
|
g = { key: `${key}#${groups.length}`, dayKey: key, label: dayLabel(iso), posts: [], artists: new Set() }
|
||||||
|
groups.push(g)
|
||||||
|
}
|
||||||
|
g.posts.push(p)
|
||||||
|
if (p.artist?.id != null) g.artists.add(p.artist.id)
|
||||||
|
}
|
||||||
|
return groups.map((g) => ({ ...g, artistCount: g.artists.size }))
|
||||||
|
})
|
||||||
|
|
||||||
// Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and
|
// Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and
|
||||||
// any active artist/platform scope) intact instead of resetting the surface.
|
// any active artist/platform scope) intact instead of resetting the surface.
|
||||||
const allPostsTarget = computed(() => {
|
const allPostsTarget = computed(() => {
|
||||||
@@ -232,6 +295,80 @@ onUnmounted(() => { teardownFeed(); teardownAround() })
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
/* Below the breakpoint: today's layout exactly — a 900px column with the
|
||||||
|
filters and ribbon inline above the feed. */
|
||||||
|
.fc-posts { max-width: 900px; }
|
||||||
|
.fc-posts__day-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: 10px;
|
||||||
|
padding: 4px 0 8px;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-posts__day-label {
|
||||||
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 1rem;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
}
|
||||||
|
.fc-posts__day-count { font-size: 0.78rem; }
|
||||||
|
|
||||||
|
/* Wide window (#407 D + E). The rail holds filters and status; each day's
|
||||||
|
heading moves into a sticky gutter beside its posts; the column widens and
|
||||||
|
the cards switch to their filmstrip layout on their own (PostCard measures
|
||||||
|
itself). 1600px is where a 280px rail and a 150px gutter still leave a card
|
||||||
|
wide enough to be worth the change. */
|
||||||
|
@media (min-width: 1600px) {
|
||||||
|
.fc-posts { max-width: 2360px; }
|
||||||
|
.fc-posts__layout {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 280px minmax(0, 1fr);
|
||||||
|
gap: 40px;
|
||||||
|
align-items: start;
|
||||||
|
}
|
||||||
|
.fc-posts__rail {
|
||||||
|
position: sticky;
|
||||||
|
top: calc(var(--fc-nav-h, 64px) + 16px);
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 16px;
|
||||||
|
}
|
||||||
|
.fc-posts__rail :deep(.fc-posts-filters) {
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: stretch;
|
||||||
|
padding-bottom: 0;
|
||||||
|
}
|
||||||
|
.fc-posts__rail :deep(.fc-posts-filters__artist),
|
||||||
|
.fc-posts__rail :deep(.fc-posts-filters__platform) {
|
||||||
|
flex: none;
|
||||||
|
width: 100%;
|
||||||
|
min-width: 0;
|
||||||
|
max-width: none;
|
||||||
|
}
|
||||||
|
.fc-posts__rail :deep(.fc-ribbon) {
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: flex-start;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.fc-posts__main { max-width: 1900px; }
|
||||||
|
.fc-posts__day {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 150px minmax(0, 1fr);
|
||||||
|
gap: 0 24px;
|
||||||
|
}
|
||||||
|
.fc-posts__day-head {
|
||||||
|
position: sticky;
|
||||||
|
top: calc(var(--fc-nav-h, 64px) + 16px);
|
||||||
|
align-self: start;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: flex-end;
|
||||||
|
gap: 2px;
|
||||||
|
padding-top: 12px;
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
.fc-posts__day-label { font-size: 1.1rem; }
|
||||||
|
}
|
||||||
|
|
||||||
.fc-posts__loading,
|
.fc-posts__loading,
|
||||||
.fc-posts__empty {
|
.fc-posts__empty {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
+28
@@ -0,0 +1,28 @@
|
|||||||
|
{
|
||||||
|
"author": "example-artist",
|
||||||
|
"author_files": [],
|
||||||
|
"author_id": "100000000000000001",
|
||||||
|
"category": "discord",
|
||||||
|
"channel": "nsfw-drops",
|
||||||
|
"channel_id": "200000000000000002",
|
||||||
|
"channel_topic": "",
|
||||||
|
"channel_type": 0,
|
||||||
|
"date": "2024-07-16 16:43:23",
|
||||||
|
"extension": "png",
|
||||||
|
"filename": "image",
|
||||||
|
"files": [],
|
||||||
|
"is_thread": false,
|
||||||
|
"message": "",
|
||||||
|
"message_id": "300000000000000003",
|
||||||
|
"num": 1,
|
||||||
|
"owner_id": "100000000000000001",
|
||||||
|
"parent": "",
|
||||||
|
"parent_id": "",
|
||||||
|
"parent_type": 0,
|
||||||
|
"server": "Example Server",
|
||||||
|
"server_files": [],
|
||||||
|
"server_id": "400000000000000004",
|
||||||
|
"subcategory": "channel",
|
||||||
|
"type": "attachment",
|
||||||
|
"url": "https://cdn.discordapp.com/attachments/200000000000000002/500000000000000005/image.png"
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""The Discord repair (#3999): forget the broken downloads, delete them, backfill again.
|
||||||
|
|
||||||
|
The danger in a repair like this is reach. It deletes files and clears
|
||||||
|
gallery-dl's memory of what it fetched, so most of these tests pin what it must
|
||||||
|
NOT touch: other platforms' archive entries, Discord server assets, images
|
||||||
|
outside a `discord/None/` folder, and a correctly named Discord file that
|
||||||
|
happens to share a folder name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.models import Artist, ImageRecord, Source
|
||||||
|
from backend.app.services import discord_repair
|
||||||
|
from backend.app.services.gallery_dl import archive_path
|
||||||
|
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
|
||||||
|
|
||||||
|
|
||||||
|
def _archive(tmp_path, entries):
|
||||||
|
path = archive_path(tmp_path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
# The table gallery-dl itself creates (gallery_dl/archive.py).
|
||||||
|
conn.execute("CREATE TABLE archive (entry TEXT PRIMARY KEY) WITHOUT ROWID")
|
||||||
|
conn.executemany("INSERT INTO archive (entry) VALUES (?)", [(e,) for e in entries])
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _entries(path):
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
try:
|
||||||
|
return sorted(r[0] for r in conn.execute("SELECT entry FROM archive"))
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# --- the archive ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_discord_message_entries_are_forgotten(tmp_path):
|
||||||
|
path = _archive(tmp_path, [
|
||||||
|
"discord300000000000000003_1",
|
||||||
|
"discord300000000000000004_2",
|
||||||
|
"discordasset_400000000000000004_9",
|
||||||
|
"hentaifoundry12345",
|
||||||
|
])
|
||||||
|
assert discord_repair.count_archive_entries(path) == 2
|
||||||
|
assert discord_repair.forget_archive_entries(path) == 2
|
||||||
|
assert _entries(path) == ["discordasset_400000000000000004_9", "hentaifoundry12345"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_counting_changes_nothing(tmp_path):
|
||||||
|
path = _archive(tmp_path, ["discord1_1", "discord2_1"])
|
||||||
|
discord_repair.count_archive_entries(path)
|
||||||
|
assert _entries(path) == ["discord1_1", "discord2_1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_missing_archive_or_table_is_zero_not_an_error(tmp_path):
|
||||||
|
assert discord_repair.count_archive_entries(tmp_path / "nope.sqlite3") == 0
|
||||||
|
empty = tmp_path / "empty.sqlite3"
|
||||||
|
sqlite3.connect(empty).close()
|
||||||
|
assert discord_repair.forget_archive_entries(empty) == 0
|
||||||
|
|
||||||
|
|
||||||
|
# --- the folders ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_broken_directories_finds_every_artist_folder_including_case_twins(tmp_path):
|
||||||
|
for artist in ("Conto", "conto", "knuxy"):
|
||||||
|
(tmp_path / artist / "discord" / "None").mkdir(parents=True)
|
||||||
|
(tmp_path / "conto" / "discord" / "general").mkdir(parents=True)
|
||||||
|
(tmp_path / "conto" / "patreon" / "None").mkdir(parents=True)
|
||||||
|
found = [
|
||||||
|
p.relative_to(tmp_path).as_posix()
|
||||||
|
for p in discord_repair.broken_directories(tmp_path)
|
||||||
|
]
|
||||||
|
assert found == ["Conto/discord/None", "conto/discord/None", "knuxy/discord/None"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_sweep_removes_leftover_sidecars_and_the_empty_folder(tmp_path):
|
||||||
|
d = tmp_path / "conto" / "discord" / "None"
|
||||||
|
d.mkdir(parents=True)
|
||||||
|
(d / "image.json").write_text("{}")
|
||||||
|
(d / "20240716_None_rejected.png").write_bytes(b"x")
|
||||||
|
removed, gone = discord_repair._sweep_directory(d)
|
||||||
|
assert (removed, gone) == (2, True)
|
||||||
|
assert not d.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# --- the whole repair, against the database -----------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _image(db_sync, artist, path, sha):
|
||||||
|
img = ImageRecord(
|
||||||
|
artist_id=artist.id, path=str(path), sha256=sha * 64, size_bytes=100,
|
||||||
|
mime="image/png", origin="downloaded",
|
||||||
|
)
|
||||||
|
db_sync.add(img)
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_repair_deletes_only_broken_discord_images_and_rearms_discord_backfills(
|
||||||
|
db_sync, tmp_path,
|
||||||
|
):
|
||||||
|
artist = Artist(name="Conto", slug="conto")
|
||||||
|
db_sync.add(artist)
|
||||||
|
db_sync.flush()
|
||||||
|
discord = Source(
|
||||||
|
artist_id=artist.id, platform="discord", url="https://discord.com/channels/1/2",
|
||||||
|
enabled=True, config_overrides={"_backfill_state": "complete", "_backfill_cursor": "x"},
|
||||||
|
)
|
||||||
|
patreon = Source(
|
||||||
|
artist_id=artist.id, platform="patreon", url="https://www.patreon.com/conto",
|
||||||
|
enabled=True, config_overrides={},
|
||||||
|
)
|
||||||
|
db_sync.add_all([discord, patreon])
|
||||||
|
|
||||||
|
broken_dir = tmp_path / "conto" / "discord" / "None"
|
||||||
|
good_dir = tmp_path / "conto" / "discord" / "general"
|
||||||
|
patreon_dir = tmp_path / "conto" / "patreon" / "None"
|
||||||
|
for d in (broken_dir, good_dir, patreon_dir):
|
||||||
|
d.mkdir(parents=True)
|
||||||
|
broken = broken_dir / "20240716_None_image.png"
|
||||||
|
good = good_dir / "20240716_300000000000000003_01_image.png"
|
||||||
|
# A `None` folder under another platform, and a `_None_` name outside one.
|
||||||
|
# Neither is the bug's pair, so neither may be touched.
|
||||||
|
elsewhere = patreon_dir / "20240716_None_image.png"
|
||||||
|
for f in (broken, good, elsewhere):
|
||||||
|
f.write_bytes(b"x")
|
||||||
|
(broken_dir / "image.json").write_text("{}")
|
||||||
|
_image(db_sync, artist, broken, "a")
|
||||||
|
_image(db_sync, artist, good, "b")
|
||||||
|
_image(db_sync, artist, elsewhere, "c")
|
||||||
|
db_sync.commit()
|
||||||
|
_archive(tmp_path, ["discord300000000000000003_1", "hentaifoundry1"])
|
||||||
|
|
||||||
|
preview = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=True)
|
||||||
|
assert preview["images"] == 1
|
||||||
|
assert preview["archive_entries"] == 1
|
||||||
|
assert broken.exists()
|
||||||
|
|
||||||
|
result = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=False)
|
||||||
|
assert result["images_deleted"] == 1
|
||||||
|
assert result["remaining"] == 0
|
||||||
|
assert result["backfills_started"] == 1
|
||||||
|
|
||||||
|
assert not broken.exists() and not broken_dir.exists()
|
||||||
|
assert good.exists() and elsewhere.exists()
|
||||||
|
assert _entries(archive_path(tmp_path)) == ["hentaifoundry1"]
|
||||||
|
|
||||||
|
kept = db_sync.execute(select(func.count(ImageRecord.id))).scalar_one()
|
||||||
|
assert kept == 2
|
||||||
|
|
||||||
|
rows = {
|
||||||
|
platform: (runs, overrides)
|
||||||
|
for platform, runs, overrides in db_sync.execute(
|
||||||
|
select(Source.platform, Source.backfill_runs_remaining, Source.config_overrides)
|
||||||
|
).all()
|
||||||
|
}
|
||||||
|
runs, overrides = rows["discord"]
|
||||||
|
assert runs == BACKFILL_MAX_CHUNKS
|
||||||
|
assert overrides["_backfill_state"] == "running"
|
||||||
|
assert "_backfill_cursor" not in overrides
|
||||||
|
# Only Discord sources are re-armed.
|
||||||
|
assert rows["patreon"][1] == {}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
"""gallery-dl naming patterns, rendered against the metadata gallery-dl really emits.
|
||||||
|
|
||||||
|
gallery-dl does not fail on a format field that names a key the extractor never
|
||||||
|
sets. It renders "None" and carries on. That is how every Discord download
|
||||||
|
landed in a `None/` directory as `<date>_None_<name>`: the patterns asked for
|
||||||
|
`{channel[name]}` and `{id}`, and the real metadata has a string `channel` and
|
||||||
|
`message_id`, with no `id` at all. The mismatch also broke sidecar pairing, so
|
||||||
|
no Discord file ever got a Post or a post date.
|
||||||
|
|
||||||
|
The fixture keeps the key set and value TYPES of a real attachment sidecar from
|
||||||
|
the operator's instance (2026-09-13), with every value invented. Rendering
|
||||||
|
through Python's own formatter raises on a missing key or a subscript into a
|
||||||
|
string, which is the loud failure gallery-dl does not give.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import string
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.app.services.gallery_dl import (
|
||||||
|
DISCORD_DIRECTORY,
|
||||||
|
DISCORD_FILENAME,
|
||||||
|
GalleryDLService,
|
||||||
|
SourceConfig,
|
||||||
|
sidecar_name_for,
|
||||||
|
)
|
||||||
|
from backend.app.utils.sidecar import find_sidecar
|
||||||
|
|
||||||
|
_FIXTURE = Path(__file__).parent / "fixtures" / "discord_attachment_sidecar.json"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def kwdict():
|
||||||
|
data = json.loads(_FIXTURE.read_text())
|
||||||
|
# gallery-dl hands the formatter a datetime; the JSON sidecar stores it as text.
|
||||||
|
data["date"] = datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _render(pattern, kwdict):
|
||||||
|
return string.Formatter().vformat(pattern, (), kwdict)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_discord_filename_renders_from_real_keys(kwdict):
|
||||||
|
name = _render(DISCORD_FILENAME, kwdict)
|
||||||
|
assert "None" not in name
|
||||||
|
assert name == "20240716_300000000000000003_01_image.png"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_discord_directory_renders_from_real_keys(kwdict):
|
||||||
|
assert [_render(p, kwdict) for p in DISCORD_DIRECTORY] == ["nsfw-drops"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"broken", ["{channel[name]}", "{date:%Y%m%d}_{id}_{filename}.{extension}"],
|
||||||
|
)
|
||||||
|
def test_the_patterns_that_shipped_would_have_failed_here(kwdict, broken):
|
||||||
|
"""Positive control: the two patterns that produced `None/..._None_...`
|
||||||
|
must fail this renderer, or the tests above prove nothing."""
|
||||||
|
with pytest.raises((KeyError, TypeError)):
|
||||||
|
_render(broken, kwdict)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_sidecar_is_named_exactly_like_its_media(kwdict, tmp_path):
|
||||||
|
"""So `find_sidecar` pairs them on its first candidate, and two attachments
|
||||||
|
that share an original name (`image.png`) can never share a sidecar."""
|
||||||
|
media = tmp_path / _render(DISCORD_FILENAME, kwdict)
|
||||||
|
sidecar = tmp_path / _render(sidecar_name_for(DISCORD_FILENAME), kwdict)
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
sidecar.write_text("{}")
|
||||||
|
assert find_sidecar(media) == sidecar
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_attachments_with_the_same_original_name_get_distinct_sidecars(kwdict):
|
||||||
|
other = {**kwdict, "message_id": "300000000000000099"}
|
||||||
|
pattern = sidecar_name_for(DISCORD_FILENAME)
|
||||||
|
assert _render(pattern, kwdict) != _render(pattern, other)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sidecar_name_for_needs_an_extension_suffix():
|
||||||
|
assert sidecar_name_for("{a}_{b}.{extension}") == "{a}_{b}.json"
|
||||||
|
assert sidecar_name_for("{a}_{b}") is None
|
||||||
|
|
||||||
|
|
||||||
|
# --- the config gallery-dl is actually given -----------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def gdl(tmp_path):
|
||||||
|
return GalleryDLService(images_root=tmp_path / "images", validate_files=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _discord_section(gdl, **overrides):
|
||||||
|
cfg = gdl._build_config_for_source(
|
||||||
|
platform="discord", source_config=SourceConfig(**overrides), artist_slug="a",
|
||||||
|
)
|
||||||
|
return cfg["extractor"]["discord"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_discord_config_carries_the_mirrored_sidecar(gdl):
|
||||||
|
section = _discord_section(gdl)
|
||||||
|
assert section["filename"] == DISCORD_FILENAME
|
||||||
|
assert section["directory"] == DISCORD_DIRECTORY
|
||||||
|
assert section["postprocessors"][0]["filename"] == sidecar_name_for(DISCORD_FILENAME)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filename_override_keeps_the_sidecar_mirrored(gdl):
|
||||||
|
section = _discord_section(gdl, filename_pattern="{message_id}_{num}.{extension}")
|
||||||
|
assert section["postprocessors"][0]["filename"] == "{message_id}_{num}.json"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_metadata_means_no_discord_postprocessor(gdl):
|
||||||
|
section = _discord_section(gdl, save_metadata=False)
|
||||||
|
assert "postprocessors" not in section
|
||||||
Reference in New Issue
Block a user