Wide-window feed, Discord downloads named and dated correctly, and a repair for the broken ones #253
@@ -475,6 +475,20 @@ async def trigger_reclaim_attachments():
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
|
||||
async def trigger_repair_discord_downloads():
|
||||
"""Clean re-download of the Discord files broken by the `None` naming
|
||||
(#3999). Body {"dry_run": bool}; dry_run is the DEFAULT, because the apply
|
||||
deletes files and makes gallery-dl forget every Discord download. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import repair_discord_downloads_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True))
|
||||
async_result = repair_discord_downloads_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
||||
async def trigger_dedup_videos():
|
||||
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Repair the Discord downloads made before the naming fix (issue #3999).
|
||||
|
||||
Until dc840fe, gallery-dl's Discord patterns asked for keys the extractor never
|
||||
emits, so every Discord download landed as
|
||||
`<artist>/discord/None/<date>_None_<original name>`, next to a sidecar named
|
||||
after the attachment's ORIGINAL name. That broke two things:
|
||||
|
||||
* **No Post, no date.** `find_sidecar` can never pair those names, so these
|
||||
files were imported as loose images with no Post, and a card shows the
|
||||
download time.
|
||||
* **No trustworthy metadata to relink from.** Every `image.png` in a channel
|
||||
wrote the same `image.json`, so the surviving sidecar describes whichever
|
||||
message was written last. The message id is gone from the filename too.
|
||||
|
||||
The operator chose a clean re-download (2026-09-13) over relinking in place:
|
||||
delete the broken files and their records, make gallery-dl forget it fetched
|
||||
them, and backfill every Discord source again under the fixed naming.
|
||||
|
||||
## Why the archive is cleared for ALL of Discord
|
||||
|
||||
gallery-dl records a download as `discord{message_id}_{num}` (upstream
|
||||
`DiscordExtractor.archive_fmt`, prefixed with the category). The broken files
|
||||
lost their message ids, so there is no way to forget one source's entries and
|
||||
not another's. Every Discord download made before the fix is broken, so
|
||||
forgetting all of them is exactly right. Anything downloaded AFTER the fix still
|
||||
exists on disk under its correct name, and gallery-dl's `skip` sees the file and
|
||||
does not fetch it again.
|
||||
|
||||
## What it does not touch
|
||||
|
||||
Discord posts FC grouped itself (#388 E2) are built from Posts, and these files
|
||||
never had one, so there is nothing grouped to unwind. Images outside a
|
||||
`discord/None/` directory are never selected: the path filter requires both the
|
||||
`None` directory and the `_None_` filename, the pair only the bug produced.
|
||||
|
||||
Operator-triggered only (Settings, preview first). Never on a beat.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import ImageRecord, Source
|
||||
from .cleanup_service import delete_images
|
||||
from .gallery_dl import archive_path
|
||||
from .source_service import arm_backfill
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# `%` and `_` are LIKE wildcards, so the literal underscores around None are
|
||||
# escaped. `________` is the eight-digit date prefix the old pattern wrote.
|
||||
_BROKEN_PATH_LIKE = r"%/discord/None/________\_None\_%"
|
||||
|
||||
# Upstream keys asset downloads as `asset_{server_id}_{id}`. FC never fetches
|
||||
# server assets, but excluding them keeps this to exactly the message
|
||||
# attachments the bug mangled.
|
||||
_ARCHIVE_SQL_MATCH = r"entry LIKE 'discord%' AND entry NOT LIKE 'discordasset\_%' ESCAPE '\'"
|
||||
_COUNT_SQL = "SELECT COUNT(*) FROM archive WHERE " + _ARCHIVE_SQL_MATCH
|
||||
_DELETE_SQL = "DELETE FROM archive WHERE " + _ARCHIVE_SQL_MATCH
|
||||
|
||||
|
||||
def broken_directories(images_root: Path) -> list[Path]:
|
||||
"""Every `<artist>/discord/None` directory. Artist folders that differ only by
|
||||
case (`Conto` and `conto`) are separate directories and both are found."""
|
||||
return sorted(d for d in Path(images_root).glob("*/discord/None") if d.is_dir())
|
||||
|
||||
|
||||
def count_archive_entries(archive: Path) -> int:
|
||||
return _archive(archive, delete=False)
|
||||
|
||||
|
||||
def forget_archive_entries(archive: Path) -> int:
|
||||
return _archive(archive, delete=True)
|
||||
|
||||
|
||||
def _archive(archive: Path, *, delete: bool) -> int:
|
||||
if not archive.is_file():
|
||||
return 0
|
||||
# A download running at the same moment holds this file briefly. Waiting
|
||||
# 30s for its lock beats failing the repair over a transient contention.
|
||||
conn = sqlite3.connect(str(archive), timeout=30)
|
||||
try:
|
||||
has_table = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='archive'"
|
||||
).fetchone()
|
||||
if not has_table:
|
||||
return 0
|
||||
if not delete:
|
||||
return conn.execute(_COUNT_SQL).fetchone()[0]
|
||||
cur = conn.execute(_DELETE_SQL)
|
||||
conn.commit()
|
||||
return cur.rowcount
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _sweep_directory(directory: Path) -> tuple[int, bool]:
|
||||
"""Remove what the record deletes left behind: the collided sidecars, plus any
|
||||
file that never became a record (a quarantined or rejected download). Then
|
||||
the directory itself, if it is empty. Returns (files removed, dir removed)."""
|
||||
removed = 0
|
||||
for f in directory.iterdir():
|
||||
if f.is_file():
|
||||
try:
|
||||
f.unlink()
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
log.warning("discord repair: could not remove %s: %s", f, exc)
|
||||
try:
|
||||
directory.rmdir()
|
||||
return removed, True
|
||||
except OSError:
|
||||
return removed, False
|
||||
|
||||
|
||||
def repair_discord_downloads(
|
||||
session: Session, *, images_root: Path, dry_run: bool,
|
||||
) -> dict:
|
||||
images_root = Path(images_root)
|
||||
archive = archive_path(images_root)
|
||||
|
||||
broken = select(ImageRecord.id, ImageRecord.size_bytes).where(
|
||||
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
|
||||
)
|
||||
rows = session.execute(broken).all()
|
||||
image_ids = [r.id for r in rows]
|
||||
directories = broken_directories(images_root)
|
||||
sources = session.execute(
|
||||
select(Source).where(Source.platform == "discord")
|
||||
).scalars().all()
|
||||
|
||||
summary = {
|
||||
"images": len(image_ids),
|
||||
"bytes": sum(r.size_bytes or 0 for r in rows),
|
||||
"directories": len(directories),
|
||||
"sources": len(sources),
|
||||
"enabled_sources": sum(1 for s in sources if s.enabled),
|
||||
}
|
||||
|
||||
if dry_run:
|
||||
summary["archive_entries"] = count_archive_entries(archive)
|
||||
return summary
|
||||
|
||||
deleted = delete_images(session, image_ids=image_ids, images_root=images_root)
|
||||
|
||||
swept = 0
|
||||
directories_removed = 0
|
||||
for d in directories:
|
||||
n, gone = _sweep_directory(d)
|
||||
swept += n
|
||||
directories_removed += int(gone)
|
||||
|
||||
# Only after the files are gone. Forgetting first and failing half way would
|
||||
# leave gallery-dl free to re-fetch into a directory still full of the old
|
||||
# copies.
|
||||
forgotten = forget_archive_entries(archive)
|
||||
|
||||
for source in sources:
|
||||
arm_backfill(source)
|
||||
session.commit()
|
||||
|
||||
remaining = session.execute(
|
||||
select(func.count(ImageRecord.id)).where(
|
||||
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
summary.update(
|
||||
images_deleted=deleted["images_deleted"],
|
||||
files_failed=deleted["files_failed"],
|
||||
leftover_files_removed=swept,
|
||||
directories_removed=directories_removed,
|
||||
archive_entries=forgotten,
|
||||
backfills_started=len(sources),
|
||||
remaining=remaining,
|
||||
)
|
||||
log.info("discord repair applied: %s", summary)
|
||||
return summary
|
||||
@@ -94,6 +94,58 @@ BACKFILL_CHUNK_SECONDS = 600
|
||||
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||
|
||||
|
||||
# --- Discord naming ---------------------------------------------------------
|
||||
#
|
||||
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
|
||||
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
|
||||
# for an attachment: `channel` is a plain STRING (the channel's name), the
|
||||
# message is `message_id`, the attachment's position in it is `num`, and there
|
||||
# is NO `id` key at all.
|
||||
#
|
||||
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
|
||||
# "None", so every Discord download since the platform was added landed in a
|
||||
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
|
||||
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
|
||||
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
|
||||
# got a Post or a post date, and (b) collides: every `image.png` in a channel
|
||||
# overwrote the same `image.json`, so the one sidecar that survived described
|
||||
# whichever message happened to be written last.
|
||||
#
|
||||
# The fix names the sidecar EXACTLY like the media minus its extension, so
|
||||
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
|
||||
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
|
||||
# these patterns against a sanitized copy of the real sidecar, so a key that
|
||||
# does not exist fails CI instead of silently becoming "None".
|
||||
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
|
||||
DISCORD_DIRECTORY = ["{channel}"]
|
||||
|
||||
|
||||
def sidecar_name_for(media_pattern: str) -> str | None:
|
||||
"""The metadata filename pattern that names a sidecar exactly like its media.
|
||||
|
||||
Returns None for a pattern that does not end in `.{extension}`, since then
|
||||
there is no media stem to mirror and the caller must fall back.
|
||||
"""
|
||||
suffix = ".{extension}"
|
||||
if not media_pattern.endswith(suffix):
|
||||
return None
|
||||
return media_pattern[: -len(suffix)] + ".json"
|
||||
|
||||
|
||||
def metadata_postprocessor(filename: str) -> dict:
|
||||
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
|
||||
|
||||
|
||||
def archive_path(images_root: Path) -> Path:
|
||||
"""gallery-dl's download archive: the record of what it has already fetched.
|
||||
|
||||
One definition, because the Discord repair (services/discord_repair.py) has
|
||||
to find the same file the downloader writes, without constructing a service
|
||||
whose __init__ creates directories.
|
||||
"""
|
||||
return Path(images_root) / ".gallery-dl" / "archive.sqlite3"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Per-source overrides loaded from Source.config_overrides JSON.
|
||||
@@ -376,8 +428,11 @@ class GalleryDLService:
|
||||
},
|
||||
"discord": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{channel[name]}"],
|
||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||||
"directory": DISCORD_DIRECTORY,
|
||||
"filename": DISCORD_FILENAME,
|
||||
# Overrides the global `{filename}.json` sidecar for this extractor
|
||||
# only — see the Discord naming note above.
|
||||
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
|
||||
"embeds": "all",
|
||||
"stickers": True,
|
||||
"reactions": False,
|
||||
@@ -402,7 +457,7 @@ class GalleryDLService:
|
||||
config = {
|
||||
"extractor": {
|
||||
"base-directory": str(self.images_root),
|
||||
"archive": str(self._config_dir / "archive.sqlite3"),
|
||||
"archive": str(archive_path(self.images_root)),
|
||||
"skip": True,
|
||||
"sleep": self._rate_limit,
|
||||
"sleep-request": max(0.5, self._rate_limit / 4),
|
||||
@@ -497,6 +552,17 @@ class GalleryDLService:
|
||||
if source_config.filename_pattern:
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
# A platform that names its sidecar after its media must keep doing so
|
||||
# under a per-source filename override, or the pairing breaks exactly the
|
||||
# way Discord's did. No metadata wanted means no platform postprocessor
|
||||
# either — the global list was already dropped above.
|
||||
if "postprocessors" in platform_section:
|
||||
mirrored = sidecar_name_for(platform_section.get("filename") or "")
|
||||
if not source_config.save_metadata or mirrored is None:
|
||||
platform_section.pop("postprocessors")
|
||||
else:
|
||||
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
|
||||
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
|
||||
@@ -158,6 +158,19 @@ def _is_app_managed(key: str) -> bool:
|
||||
BACKFILL_MAX_CHUNKS = 200
|
||||
|
||||
|
||||
def arm_backfill(source: Source) -> None:
|
||||
"""Arm a fresh run-until-done backfill on `source` (plan #693). Mutation
|
||||
only — the caller commits. Shared by `SourceService.start_backfill` and the
|
||||
sync Discord repair, so both clear exactly the same resume state."""
|
||||
co = dict(source.config_overrides or {})
|
||||
co["_backfill_state"] = "running"
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||
|
||||
|
||||
class SourceService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
@@ -473,13 +486,7 @@ class SourceService:
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
co = dict(source.config_overrides or {})
|
||||
co["_backfill_state"] = "running"
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||
arm_backfill(source)
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
|
||||
@@ -122,6 +122,28 @@ def prune_missing_file_records_task(self) -> dict:
|
||||
return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.repair_discord_downloads_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def repair_discord_downloads_task(self, dry_run: bool = True) -> dict:
|
||||
"""Clean re-download of the Discord files broken by the `None` naming
|
||||
(#3999). dry_run (the default) returns the projection; apply deletes the
|
||||
broken images and their files, makes gallery-dl forget every Discord
|
||||
download, and restarts every Discord source's backfill. Defaults to the SAFE
|
||||
preview because the apply deletes files. Operator-triggered only."""
|
||||
from ..services.discord_repair import repair_discord_downloads
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return repair_discord_downloads(
|
||||
session, images_root=IMAGES_ROOT, dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.dedup_videos_task",
|
||||
bind=True,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
<template>
|
||||
<v-card class="fc-post-card" variant="outlined">
|
||||
<v-card
|
||||
ref="cardEl" class="fc-post-card" variant="outlined"
|
||||
:class="{ 'fc-post-card--wide': wide }"
|
||||
>
|
||||
<div class="fc-post-card__head">
|
||||
<!-- Posts with no live subscription have source=null (alembic 0030);
|
||||
show a "filesystem import" affordance instead of a platform chip. -->
|
||||
@@ -58,7 +61,7 @@
|
||||
</button>
|
||||
<div
|
||||
v-if="rail.length || moreCount" class="fc-post-card__rail"
|
||||
:style="{ '--fc-rail-cols': railCols }"
|
||||
:style="{ '--fc-rail-cols': railCols, '--fc-grid-cols': Math.min(2, railCols) }"
|
||||
>
|
||||
<button
|
||||
v-for="t in rail" :key="t.image_id" type="button"
|
||||
@@ -221,20 +224,33 @@ const synthesisTitle = computed(() => {
|
||||
})
|
||||
|
||||
const hero = computed(() => images.value[0])
|
||||
// The thumbnail strip spans the hero's full width (CSS grid, equal columns),
|
||||
// rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are
|
||||
// more images than fit, the last cell becomes a "+N" overflow tile so the
|
||||
// count stays accurate.
|
||||
const RAIL_MAX = 5
|
||||
|
||||
// Filmstrip layout (milestone #407, option A). On a very wide window a card
|
||||
// that just grew would give one post the whole screen, so a WIDE card instead
|
||||
// pins the hero to a fixed height and moves the extra images into a 2-column
|
||||
// grid BESIDE it. Measured on the card rather than the viewport because the
|
||||
// same card renders in the Latest feed, Browse, and the in-context view, each
|
||||
// at a different width.
|
||||
const WIDE_CARD_PX = 1100
|
||||
const cardEl = ref(null)
|
||||
const wide = ref(false)
|
||||
|
||||
// The narrow layout's strip spans the hero's full width (CSS grid, equal
|
||||
// columns); the wide layout's grid is 2×2. Show up to that many cells; when
|
||||
// there are more images than fit, the last cell becomes a "+N" overflow tile so
|
||||
// the count stays accurate.
|
||||
const RAIL_MAX_NARROW = 5
|
||||
const RAIL_MAX_WIDE = 4
|
||||
const serverMore = computed(() => props.post.thumbnails_more || 0)
|
||||
const afterHero = computed(() => images.value.slice(1))
|
||||
const railMax = computed(() => (wide.value ? RAIL_MAX_WIDE : RAIL_MAX_NARROW))
|
||||
const hasOverflow = computed(
|
||||
() => serverMore.value > 0 || afterHero.value.length > RAIL_MAX,
|
||||
() => serverMore.value > 0 || afterHero.value.length > railMax.value,
|
||||
)
|
||||
const rail = computed(() =>
|
||||
hasOverflow.value
|
||||
? afterHero.value.slice(0, RAIL_MAX - 1)
|
||||
: afterHero.value.slice(0, RAIL_MAX),
|
||||
? afterHero.value.slice(0, railMax.value - 1)
|
||||
: afterHero.value.slice(0, railMax.value),
|
||||
)
|
||||
const visibleCount = computed(() => (images.value.length ? 1 + rail.value.length : 0))
|
||||
const moreCount = computed(() => {
|
||||
@@ -349,8 +365,17 @@ function measureOverflow () {
|
||||
}
|
||||
|
||||
let ro = null
|
||||
let cardRo = null
|
||||
onMounted(() => {
|
||||
nextTick(measureOverflow)
|
||||
const root = cardEl.value?.$el
|
||||
if (typeof ResizeObserver !== 'undefined' && root) {
|
||||
cardRo = new ResizeObserver((entries) => {
|
||||
const w = entries[0]?.contentRect?.width ?? 0
|
||||
wide.value = w >= WIDE_CARD_PX
|
||||
})
|
||||
cardRo.observe(root)
|
||||
}
|
||||
// Re-measure when the card resizes (the container-query clamp differs by
|
||||
// width). Guarded for happy-dom / older runtimes without ResizeObserver.
|
||||
if (typeof ResizeObserver !== 'undefined' && descEl.value) {
|
||||
@@ -358,7 +383,10 @@ onMounted(() => {
|
||||
ro.observe(descEl.value)
|
||||
}
|
||||
})
|
||||
onBeforeUnmount(() => { if (ro) { ro.disconnect(); ro = null } })
|
||||
onBeforeUnmount(() => {
|
||||
if (ro) { ro.disconnect(); ro = null }
|
||||
if (cardRo) { cardRo.disconnect(); cardRo = null }
|
||||
})
|
||||
|
||||
async function toggleDesc () {
|
||||
if (!descExpanded.value) {
|
||||
@@ -495,6 +523,40 @@ function formatBytes (n) {
|
||||
color: rgb(var(--v-theme-accent));
|
||||
}
|
||||
|
||||
/* --- Filmstrip layout (wide cards, #407 A) ---------------------------------
|
||||
The hero has a HEIGHT, not a width: a wide card must not turn into a
|
||||
full-screen post, so its height stays roughly a third of the viewport
|
||||
whatever the window's width. The extra images sit beside it as square cells
|
||||
whose size derives from that same height, so the grid always ends flush with
|
||||
the hero's bottom edge. */
|
||||
.fc-post-card--wide {
|
||||
--fc-hero-h: clamp(260px, 34vh, 460px);
|
||||
--fc-grid-gap: 8px;
|
||||
--fc-cell: calc((var(--fc-hero-h) - var(--fc-grid-gap)) / 2);
|
||||
}
|
||||
.fc-post-card--wide .fc-post-card__body { flex-direction: row; gap: 24px; }
|
||||
.fc-post-card--wide .fc-post-card__media {
|
||||
flex: 0 0 auto;
|
||||
display: flex;
|
||||
gap: var(--fc-grid-gap);
|
||||
}
|
||||
.fc-post-card--wide .fc-post-card__hero {
|
||||
width: auto;
|
||||
height: var(--fc-hero-h);
|
||||
}
|
||||
.fc-post-card--wide .fc-post-card__rail {
|
||||
margin-top: 0;
|
||||
gap: var(--fc-grid-gap);
|
||||
grid-template-columns: repeat(var(--fc-grid-cols, 2), var(--fc-cell));
|
||||
grid-auto-rows: var(--fc-cell);
|
||||
}
|
||||
.fc-post-card--wide .fc-post-card__rail-cell,
|
||||
.fc-post-card--wide .fc-post-card__rail-more { height: 100%; }
|
||||
.fc-post-card--wide .fc-post-card__text { flex: 1 1 0; min-width: 0; }
|
||||
/* Text is secondary here — long reads happen in the expanded view — so the
|
||||
clamp keeps the text column no taller than the images beside it. */
|
||||
.fc-post-card--wide .fc-post-card__desc--clamped { -webkit-line-clamp: 4; }
|
||||
|
||||
.fc-post-card__title {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 18px; font-weight: 700;
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
<template>
|
||||
<!-- #3999: Discord downloads made before the naming fix landed in a `None`
|
||||
folder with no post and a download-time date. This re-downloads them
|
||||
cleanly. Preview first; the apply deletes files. -->
|
||||
<MaintenanceTile
|
||||
icon="mdi-download-off-outline"
|
||||
title="Repair Discord downloads"
|
||||
blurb="Re-download Discord images that were saved with no post and the wrong date."
|
||||
destructive
|
||||
:open="applying || previewing"
|
||||
>
|
||||
<p class="text-body-2 mb-3">
|
||||
Before the naming fix, every Discord download was saved into a folder
|
||||
called <code>None</code>. Those images never got a post, so they show the
|
||||
time Curator downloaded them rather than when they were posted.
|
||||
<strong>Apply</strong> deletes those images, clears Discord from the
|
||||
download history, and starts a fresh backfill of every Discord source so
|
||||
they come back with their posts and dates. Tags you added by hand to those
|
||||
images are lost. Nothing outside a Discord <code>None</code> folder is
|
||||
touched.
|
||||
</p>
|
||||
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" variant="tonal" rounded="pill"
|
||||
:loading="previewing" :disabled="applying" @click="preview"
|
||||
>
|
||||
<v-icon start>mdi-magnify</v-icon> Preview
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="error" rounded="pill"
|
||||
:loading="applying"
|
||||
:disabled="previewing || !canApply"
|
||||
@click="confirmOpen = true"
|
||||
>
|
||||
<v-icon start>mdi-download-off-outline</v-icon> Apply
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||
density="comfortable"
|
||||
>
|
||||
<span v-if="applied">
|
||||
Deleted {{ summary.images_deleted }} image(s) and cleared
|
||||
{{ summary.archive_entries }} Discord download record(s). Backfills
|
||||
started on {{ summary.backfills_started }} Discord source(s).
|
||||
</span>
|
||||
<span v-else-if="hasWork">
|
||||
{{ summary.images }} broken image(s) ({{ humanBytes(summary.bytes) }})
|
||||
across {{ summary.directories }} folder(s), and
|
||||
{{ summary.archive_entries }} Discord download record(s) to clear.
|
||||
{{ summary.sources }} Discord source(s) will backfill again.
|
||||
</span>
|
||||
<span v-else>Nothing to repair — no broken Discord downloads found.</span>
|
||||
|
||||
<div v-if="applied && summary.files_failed" class="mt-1 text-caption">
|
||||
{{ summary.files_failed }} file(s) could not be removed — see the worker log.
|
||||
</div>
|
||||
<div v-if="applied && summary.remaining" class="mt-1 text-caption">
|
||||
{{ summary.remaining }} broken image(s) are still present. Run it again.
|
||||
</div>
|
||||
</v-alert>
|
||||
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Maintenance" />
|
||||
|
||||
<v-dialog v-model="confirmOpen" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title>Repair Discord downloads?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
This permanently deletes <strong>{{ summary?.images ?? 0 }}</strong>
|
||||
Discord image(s) ({{ humanBytes(summary?.bytes) }}) and re-downloads
|
||||
them from Discord with their posts and dates. Any tags you added to
|
||||
those images by hand will not come back.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||
<v-btn color="error" @click="apply">Repair</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
|
||||
endpoint: '/api/admin/maintenance/repair-discord-downloads',
|
||||
storageKey: 'fc.maint.repairDiscordDownloads',
|
||||
appliedToast: 'Discord downloads cleared — backfills started',
|
||||
})
|
||||
|
||||
// The archive count matters too: files could already be gone while gallery-dl
|
||||
// still believes it has them, which would stop the backfill re-fetching.
|
||||
const hasWork = computed(
|
||||
() => !!summary.value && (summary.value.images > 0 || summary.value.archive_entries > 0),
|
||||
)
|
||||
const canApply = computed(() => hasWork.value && !applied.value)
|
||||
const summaryType = computed(() => {
|
||||
if (applied.value) return 'success'
|
||||
return hasWork.value ? 'info' : 'success'
|
||||
})
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
applyTask()
|
||||
}
|
||||
</script>
|
||||
@@ -15,6 +15,7 @@
|
||||
<ImportFiltersForm />
|
||||
<TranslationCard />
|
||||
<DiscordGroupingCard />
|
||||
<DiscordRepairCard />
|
||||
<PostAssociationsCard />
|
||||
<MembershipRosterCard />
|
||||
<MembershipSuggestionsCard />
|
||||
@@ -85,6 +86,7 @@ import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
||||
import CropProposersCard from './CropProposersCard.vue'
|
||||
import HeadsCard from './HeadsCard.vue'
|
||||
import DiscordGroupingCard from './DiscordGroupingCard.vue'
|
||||
import DiscordRepairCard from './DiscordRepairCard.vue'
|
||||
import MembershipRosterCard from './MembershipRosterCard.vue'
|
||||
import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue'
|
||||
import PostAssociationsCard from './PostAssociationsCard.vue'
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
<template>
|
||||
<v-container class="pt-2 pb-6" max-width="900">
|
||||
<!-- Width is set in CSS, not with `max-width` here: below 1600px it is
|
||||
today's 900px column, and above it the feed widens and gains the rail
|
||||
and day gutter (milestone #407). -->
|
||||
<v-container fluid class="pt-2 pb-6 fc-posts">
|
||||
<!-- In-context view: deep-linked to one post, with bidirectional infinite
|
||||
scroll — newer posts load above, older posts below. -->
|
||||
<template v-if="postIdFilter != null">
|
||||
@@ -48,15 +51,19 @@
|
||||
</template>
|
||||
|
||||
<!-- Normal feed -->
|
||||
<template v-else>
|
||||
<FeedStatusRibbon v-if="statusRibbon" />
|
||||
|
||||
<PostsFilterBar
|
||||
:artist-id="artistFilter"
|
||||
:platform="platformFilter"
|
||||
@update:filters="onFilters"
|
||||
/>
|
||||
<div v-else class="fc-posts__layout">
|
||||
<!-- On a wide window this is a sticky left rail (#407 E); below the
|
||||
breakpoint it lays out exactly as the old inline header did. -->
|
||||
<aside class="fc-posts__rail">
|
||||
<PostsFilterBar
|
||||
:artist-id="artistFilter"
|
||||
:platform="platformFilter"
|
||||
@update:filters="onFilters"
|
||||
/>
|
||||
<FeedStatusRibbon v-if="statusRibbon" />
|
||||
</aside>
|
||||
|
||||
<div class="fc-posts__main">
|
||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||
{{ String(store.error) }}
|
||||
</v-alert>
|
||||
@@ -74,14 +81,28 @@
|
||||
</div>
|
||||
|
||||
<div v-else>
|
||||
<PostCard v-for="p in store.items" :key="p.id" :post="p" />
|
||||
<!-- Day groups (#407 D). The heading sits above its posts on a narrow
|
||||
window and in a sticky left gutter on a wide one. -->
|
||||
<section v-for="d in days" :key="d.key" class="fc-posts__day">
|
||||
<header class="fc-posts__day-head">
|
||||
<span class="fc-posts__day-label">{{ d.label }}</span>
|
||||
<span class="fc-posts__day-count">
|
||||
{{ d.posts.length }} post{{ d.posts.length === 1 ? '' : 's' }}
|
||||
· {{ d.artistCount }} artist{{ d.artistCount === 1 ? '' : 's' }}
|
||||
</span>
|
||||
</header>
|
||||
<div class="fc-posts__day-posts">
|
||||
<PostCard v-for="p in d.posts" :key="p.id" :post="p" />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div ref="sentinel" class="fc-posts__sentinel">
|
||||
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
|
||||
<span v-else-if="store.done" class="fc-posts__end">End of stream</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
@@ -121,6 +142,48 @@ const hasActiveFilter = computed(() =>
|
||||
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
|
||||
)
|
||||
|
||||
// --- day groups (#407 D) ---
|
||||
// CONSECUTIVE runs, not a bucket per date. The feed's sort key includes
|
||||
// `resurfaced_at` (a Discord grouping that grew moves back to the top), which
|
||||
// the payload does not carry, so a resurfaced post can sit above newer ones.
|
||||
// Bucketing by date would pull it out of order; a run gives it its own heading
|
||||
// where it actually appears. Counts cover what has LOADED, and grow as the
|
||||
// infinite scroll fetches more of the same day.
|
||||
function dayKey (iso) {
|
||||
const d = new Date(iso)
|
||||
return `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`
|
||||
}
|
||||
function dayLabel (iso) {
|
||||
const d = new Date(iso)
|
||||
const today = new Date()
|
||||
const startOf = (x) => new Date(x.getFullYear(), x.getMonth(), x.getDate()).getTime()
|
||||
const days = Math.round((startOf(today) - startOf(d)) / 86400000)
|
||||
if (days === 0) return 'Today'
|
||||
if (days === 1) return 'Yesterday'
|
||||
if (days > 1 && days < 7) return d.toLocaleDateString(undefined, { weekday: 'long' })
|
||||
const sameYear = d.getFullYear() === today.getFullYear()
|
||||
return d.toLocaleDateString(undefined, {
|
||||
month: 'short', day: 'numeric', ...(sameYear ? {} : { year: 'numeric' }),
|
||||
})
|
||||
}
|
||||
const days = computed(() => {
|
||||
const groups = []
|
||||
for (const p of store.items) {
|
||||
const iso = p.post_date || p.downloaded_at
|
||||
const key = dayKey(iso)
|
||||
let g = groups[groups.length - 1]
|
||||
if (!g || g.dayKey !== key) {
|
||||
// Suffix with the run index so a day that appears twice (see above)
|
||||
// still has a unique v-for key.
|
||||
g = { key: `${key}#${groups.length}`, dayKey: key, label: dayLabel(iso), posts: [], artists: new Set() }
|
||||
groups.push(g)
|
||||
}
|
||||
g.posts.push(p)
|
||||
if (p.artist?.id != null) g.artists.add(p.artist.id)
|
||||
}
|
||||
return groups.map((g) => ({ ...g, artistCount: g.artists.size }))
|
||||
})
|
||||
|
||||
// Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and
|
||||
// any active artist/platform scope) intact instead of resetting the surface.
|
||||
const allPostsTarget = computed(() => {
|
||||
@@ -232,6 +295,80 @@ onUnmounted(() => { teardownFeed(); teardownAround() })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* Below the breakpoint: today's layout exactly — a 900px column with the
|
||||
filters and ribbon inline above the feed. */
|
||||
.fc-posts { max-width: 900px; }
|
||||
.fc-posts__day-head {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 10px;
|
||||
padding: 4px 0 8px;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-posts__day-label {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-weight: 600;
|
||||
font-size: 1rem;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
}
|
||||
.fc-posts__day-count { font-size: 0.78rem; }
|
||||
|
||||
/* Wide window (#407 D + E). The rail holds filters and status; each day's
|
||||
heading moves into a sticky gutter beside its posts; the column widens and
|
||||
the cards switch to their filmstrip layout on their own (PostCard measures
|
||||
itself). 1600px is where a 280px rail and a 150px gutter still leave a card
|
||||
wide enough to be worth the change. */
|
||||
@media (min-width: 1600px) {
|
||||
.fc-posts { max-width: 2360px; }
|
||||
.fc-posts__layout {
|
||||
display: grid;
|
||||
grid-template-columns: 280px minmax(0, 1fr);
|
||||
gap: 40px;
|
||||
align-items: start;
|
||||
}
|
||||
.fc-posts__rail {
|
||||
position: sticky;
|
||||
top: calc(var(--fc-nav-h, 64px) + 16px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
.fc-posts__rail :deep(.fc-posts-filters) {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
padding-bottom: 0;
|
||||
}
|
||||
.fc-posts__rail :deep(.fc-posts-filters__artist),
|
||||
.fc-posts__rail :deep(.fc-posts-filters__platform) {
|
||||
flex: none;
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
max-width: none;
|
||||
}
|
||||
.fc-posts__rail :deep(.fc-ribbon) {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
}
|
||||
.fc-posts__main { max-width: 1900px; }
|
||||
.fc-posts__day {
|
||||
display: grid;
|
||||
grid-template-columns: 150px minmax(0, 1fr);
|
||||
gap: 0 24px;
|
||||
}
|
||||
.fc-posts__day-head {
|
||||
position: sticky;
|
||||
top: calc(var(--fc-nav-h, 64px) + 16px);
|
||||
align-self: start;
|
||||
flex-direction: column;
|
||||
align-items: flex-end;
|
||||
gap: 2px;
|
||||
padding-top: 12px;
|
||||
text-align: right;
|
||||
}
|
||||
.fc-posts__day-label { font-size: 1.1rem; }
|
||||
}
|
||||
|
||||
.fc-posts__loading,
|
||||
.fc-posts__empty {
|
||||
display: flex;
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"author": "example-artist",
|
||||
"author_files": [],
|
||||
"author_id": "100000000000000001",
|
||||
"category": "discord",
|
||||
"channel": "nsfw-drops",
|
||||
"channel_id": "200000000000000002",
|
||||
"channel_topic": "",
|
||||
"channel_type": 0,
|
||||
"date": "2024-07-16 16:43:23",
|
||||
"extension": "png",
|
||||
"filename": "image",
|
||||
"files": [],
|
||||
"is_thread": false,
|
||||
"message": "",
|
||||
"message_id": "300000000000000003",
|
||||
"num": 1,
|
||||
"owner_id": "100000000000000001",
|
||||
"parent": "",
|
||||
"parent_id": "",
|
||||
"parent_type": 0,
|
||||
"server": "Example Server",
|
||||
"server_files": [],
|
||||
"server_id": "400000000000000004",
|
||||
"subcategory": "channel",
|
||||
"type": "attachment",
|
||||
"url": "https://cdn.discordapp.com/attachments/200000000000000002/500000000000000005/image.png"
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
"""The Discord repair (#3999): forget the broken downloads, delete them, backfill again.
|
||||
|
||||
The danger in a repair like this is reach. It deletes files and clears
|
||||
gallery-dl's memory of what it fetched, so most of these tests pin what it must
|
||||
NOT touch: other platforms' archive entries, Discord server assets, images
|
||||
outside a `discord/None/` folder, and a correctly named Discord file that
|
||||
happens to share a folder name.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Source
|
||||
from backend.app.services import discord_repair
|
||||
from backend.app.services.gallery_dl import archive_path
|
||||
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
|
||||
def _archive(tmp_path, entries):
|
||||
path = archive_path(tmp_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
# The table gallery-dl itself creates (gallery_dl/archive.py).
|
||||
conn.execute("CREATE TABLE archive (entry TEXT PRIMARY KEY) WITHOUT ROWID")
|
||||
conn.executemany("INSERT INTO archive (entry) VALUES (?)", [(e,) for e in entries])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return path
|
||||
|
||||
|
||||
def _entries(path):
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
return sorted(r[0] for r in conn.execute("SELECT entry FROM archive"))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# --- the archive ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_only_discord_message_entries_are_forgotten(tmp_path):
|
||||
path = _archive(tmp_path, [
|
||||
"discord300000000000000003_1",
|
||||
"discord300000000000000004_2",
|
||||
"discordasset_400000000000000004_9",
|
||||
"hentaifoundry12345",
|
||||
])
|
||||
assert discord_repair.count_archive_entries(path) == 2
|
||||
assert discord_repair.forget_archive_entries(path) == 2
|
||||
assert _entries(path) == ["discordasset_400000000000000004_9", "hentaifoundry12345"]
|
||||
|
||||
|
||||
def test_counting_changes_nothing(tmp_path):
|
||||
path = _archive(tmp_path, ["discord1_1", "discord2_1"])
|
||||
discord_repair.count_archive_entries(path)
|
||||
assert _entries(path) == ["discord1_1", "discord2_1"]
|
||||
|
||||
|
||||
def test_a_missing_archive_or_table_is_zero_not_an_error(tmp_path):
|
||||
assert discord_repair.count_archive_entries(tmp_path / "nope.sqlite3") == 0
|
||||
empty = tmp_path / "empty.sqlite3"
|
||||
sqlite3.connect(empty).close()
|
||||
assert discord_repair.forget_archive_entries(empty) == 0
|
||||
|
||||
|
||||
# --- the folders ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_broken_directories_finds_every_artist_folder_including_case_twins(tmp_path):
|
||||
for artist in ("Conto", "conto", "knuxy"):
|
||||
(tmp_path / artist / "discord" / "None").mkdir(parents=True)
|
||||
(tmp_path / "conto" / "discord" / "general").mkdir(parents=True)
|
||||
(tmp_path / "conto" / "patreon" / "None").mkdir(parents=True)
|
||||
found = [
|
||||
p.relative_to(tmp_path).as_posix()
|
||||
for p in discord_repair.broken_directories(tmp_path)
|
||||
]
|
||||
assert found == ["Conto/discord/None", "conto/discord/None", "knuxy/discord/None"]
|
||||
|
||||
|
||||
def test_sweep_removes_leftover_sidecars_and_the_empty_folder(tmp_path):
|
||||
d = tmp_path / "conto" / "discord" / "None"
|
||||
d.mkdir(parents=True)
|
||||
(d / "image.json").write_text("{}")
|
||||
(d / "20240716_None_rejected.png").write_bytes(b"x")
|
||||
removed, gone = discord_repair._sweep_directory(d)
|
||||
assert (removed, gone) == (2, True)
|
||||
assert not d.exists()
|
||||
|
||||
|
||||
# --- the whole repair, against the database -----------------------------------
|
||||
|
||||
|
||||
def _image(db_sync, artist, path, sha):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=str(path), sha256=sha * 64, size_bytes=100,
|
||||
mime="image/png", origin="downloaded",
|
||||
)
|
||||
db_sync.add(img)
|
||||
return img
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_repair_deletes_only_broken_discord_images_and_rearms_discord_backfills(
|
||||
db_sync, tmp_path,
|
||||
):
|
||||
artist = Artist(name="Conto", slug="conto")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
discord = Source(
|
||||
artist_id=artist.id, platform="discord", url="https://discord.com/channels/1/2",
|
||||
enabled=True, config_overrides={"_backfill_state": "complete", "_backfill_cursor": "x"},
|
||||
)
|
||||
patreon = Source(
|
||||
artist_id=artist.id, platform="patreon", url="https://www.patreon.com/conto",
|
||||
enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add_all([discord, patreon])
|
||||
|
||||
broken_dir = tmp_path / "conto" / "discord" / "None"
|
||||
good_dir = tmp_path / "conto" / "discord" / "general"
|
||||
patreon_dir = tmp_path / "conto" / "patreon" / "None"
|
||||
for d in (broken_dir, good_dir, patreon_dir):
|
||||
d.mkdir(parents=True)
|
||||
broken = broken_dir / "20240716_None_image.png"
|
||||
good = good_dir / "20240716_300000000000000003_01_image.png"
|
||||
# A `None` folder under another platform, and a `_None_` name outside one.
|
||||
# Neither is the bug's pair, so neither may be touched.
|
||||
elsewhere = patreon_dir / "20240716_None_image.png"
|
||||
for f in (broken, good, elsewhere):
|
||||
f.write_bytes(b"x")
|
||||
(broken_dir / "image.json").write_text("{}")
|
||||
_image(db_sync, artist, broken, "a")
|
||||
_image(db_sync, artist, good, "b")
|
||||
_image(db_sync, artist, elsewhere, "c")
|
||||
db_sync.commit()
|
||||
_archive(tmp_path, ["discord300000000000000003_1", "hentaifoundry1"])
|
||||
|
||||
preview = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=True)
|
||||
assert preview["images"] == 1
|
||||
assert preview["archive_entries"] == 1
|
||||
assert broken.exists()
|
||||
|
||||
result = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=False)
|
||||
assert result["images_deleted"] == 1
|
||||
assert result["remaining"] == 0
|
||||
assert result["backfills_started"] == 1
|
||||
|
||||
assert not broken.exists() and not broken_dir.exists()
|
||||
assert good.exists() and elsewhere.exists()
|
||||
assert _entries(archive_path(tmp_path)) == ["hentaifoundry1"]
|
||||
|
||||
kept = db_sync.execute(select(func.count(ImageRecord.id))).scalar_one()
|
||||
assert kept == 2
|
||||
|
||||
rows = {
|
||||
platform: (runs, overrides)
|
||||
for platform, runs, overrides in db_sync.execute(
|
||||
select(Source.platform, Source.backfill_runs_remaining, Source.config_overrides)
|
||||
).all()
|
||||
}
|
||||
runs, overrides = rows["discord"]
|
||||
assert runs == BACKFILL_MAX_CHUNKS
|
||||
assert overrides["_backfill_state"] == "running"
|
||||
assert "_backfill_cursor" not in overrides
|
||||
# Only Discord sources are re-armed.
|
||||
assert rows["patreon"][1] == {}
|
||||
@@ -0,0 +1,117 @@
|
||||
"""gallery-dl naming patterns, rendered against the metadata gallery-dl really emits.
|
||||
|
||||
gallery-dl does not fail on a format field that names a key the extractor never
|
||||
sets. It renders "None" and carries on. That is how every Discord download
|
||||
landed in a `None/` directory as `<date>_None_<name>`: the patterns asked for
|
||||
`{channel[name]}` and `{id}`, and the real metadata has a string `channel` and
|
||||
`message_id`, with no `id` at all. The mismatch also broke sidecar pairing, so
|
||||
no Discord file ever got a Post or a post date.
|
||||
|
||||
The fixture keeps the key set and value TYPES of a real attachment sidecar from
|
||||
the operator's instance (2026-09-13), with every value invented. Rendering
|
||||
through Python's own formatter raises on a missing key or a subscript into a
|
||||
string, which is the loud failure gallery-dl does not give.
|
||||
"""
|
||||
|
||||
import json
|
||||
import string
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.gallery_dl import (
|
||||
DISCORD_DIRECTORY,
|
||||
DISCORD_FILENAME,
|
||||
GalleryDLService,
|
||||
SourceConfig,
|
||||
sidecar_name_for,
|
||||
)
|
||||
from backend.app.utils.sidecar import find_sidecar
|
||||
|
||||
_FIXTURE = Path(__file__).parent / "fixtures" / "discord_attachment_sidecar.json"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kwdict():
|
||||
data = json.loads(_FIXTURE.read_text())
|
||||
# gallery-dl hands the formatter a datetime; the JSON sidecar stores it as text.
|
||||
data["date"] = datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
|
||||
return data
|
||||
|
||||
|
||||
def _render(pattern, kwdict):
|
||||
return string.Formatter().vformat(pattern, (), kwdict)
|
||||
|
||||
|
||||
def test_the_discord_filename_renders_from_real_keys(kwdict):
|
||||
name = _render(DISCORD_FILENAME, kwdict)
|
||||
assert "None" not in name
|
||||
assert name == "20240716_300000000000000003_01_image.png"
|
||||
|
||||
|
||||
def test_the_discord_directory_renders_from_real_keys(kwdict):
|
||||
assert [_render(p, kwdict) for p in DISCORD_DIRECTORY] == ["nsfw-drops"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"broken", ["{channel[name]}", "{date:%Y%m%d}_{id}_{filename}.{extension}"],
|
||||
)
|
||||
def test_the_patterns_that_shipped_would_have_failed_here(kwdict, broken):
|
||||
"""Positive control: the two patterns that produced `None/..._None_...`
|
||||
must fail this renderer, or the tests above prove nothing."""
|
||||
with pytest.raises((KeyError, TypeError)):
|
||||
_render(broken, kwdict)
|
||||
|
||||
|
||||
def test_the_sidecar_is_named_exactly_like_its_media(kwdict, tmp_path):
|
||||
"""So `find_sidecar` pairs them on its first candidate, and two attachments
|
||||
that share an original name (`image.png`) can never share a sidecar."""
|
||||
media = tmp_path / _render(DISCORD_FILENAME, kwdict)
|
||||
sidecar = tmp_path / _render(sidecar_name_for(DISCORD_FILENAME), kwdict)
|
||||
media.write_bytes(b"x")
|
||||
sidecar.write_text("{}")
|
||||
assert find_sidecar(media) == sidecar
|
||||
|
||||
|
||||
def test_two_attachments_with_the_same_original_name_get_distinct_sidecars(kwdict):
|
||||
other = {**kwdict, "message_id": "300000000000000099"}
|
||||
pattern = sidecar_name_for(DISCORD_FILENAME)
|
||||
assert _render(pattern, kwdict) != _render(pattern, other)
|
||||
|
||||
|
||||
def test_sidecar_name_for_needs_an_extension_suffix():
|
||||
assert sidecar_name_for("{a}_{b}.{extension}") == "{a}_{b}.json"
|
||||
assert sidecar_name_for("{a}_{b}") is None
|
||||
|
||||
|
||||
# --- the config gallery-dl is actually given -----------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdl(tmp_path):
|
||||
return GalleryDLService(images_root=tmp_path / "images", validate_files=False)
|
||||
|
||||
|
||||
def _discord_section(gdl, **overrides):
|
||||
cfg = gdl._build_config_for_source(
|
||||
platform="discord", source_config=SourceConfig(**overrides), artist_slug="a",
|
||||
)
|
||||
return cfg["extractor"]["discord"]
|
||||
|
||||
|
||||
def test_discord_config_carries_the_mirrored_sidecar(gdl):
|
||||
section = _discord_section(gdl)
|
||||
assert section["filename"] == DISCORD_FILENAME
|
||||
assert section["directory"] == DISCORD_DIRECTORY
|
||||
assert section["postprocessors"][0]["filename"] == sidecar_name_for(DISCORD_FILENAME)
|
||||
|
||||
|
||||
def test_a_filename_override_keeps_the_sidecar_mirrored(gdl):
|
||||
section = _discord_section(gdl, filename_pattern="{message_id}_{num}.{extension}")
|
||||
assert section["postprocessors"][0]["filename"] == "{message_id}_{num}.json"
|
||||
|
||||
|
||||
def test_no_metadata_means_no_discord_postprocessor(gdl):
|
||||
section = _discord_section(gdl, save_metadata=False)
|
||||
assert "postprocessors" not in section
|
||||
Reference in New Issue
Block a user