Wide-window feed, Discord downloads named and dated correctly, and a repair for the broken ones #253

Merged
bvandeusen merged 3 commits from dev into main 2026-09-13 20:57:06 -04:00
12 changed files with 958 additions and 32 deletions
+14
View File
@@ -475,6 +475,20 @@ async def trigger_reclaim_attachments():
return _queued(async_result) return _queued(async_result)
@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
async def trigger_repair_discord_downloads():
"""Clean re-download of the Discord files broken by the `None` naming
(#3999). Body {"dry_run": bool}; dry_run is the DEFAULT, because the apply
deletes files and makes gallery-dl forget every Discord download. Returns the
Celery task id — poll /maintenance/task-result/<id> for the summary."""
from ..tasks.admin import repair_discord_downloads_task
body = await request.get_json(silent=True) or {}
dry_run = bool(body.get("dry_run", True))
async_result = repair_discord_downloads_task.delay(dry_run=dry_run)
return _queued(async_result)
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"]) @admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
async def trigger_dedup_videos(): async def trigger_dedup_videos():
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
+183
View File
@@ -0,0 +1,183 @@
"""Repair the Discord downloads made before the naming fix (issue #3999).
Until dc840fe, gallery-dl's Discord patterns asked for keys the extractor never
emits, so every Discord download landed as
`<artist>/discord/None/<date>_None_<original name>`, next to a sidecar named
after the attachment's ORIGINAL name. That broke two things:
* **No Post, no date.** `find_sidecar` can never pair those names, so these
files were imported as loose images with no Post, and a card shows the
download time.
* **No trustworthy metadata to relink from.** Every `image.png` in a channel
wrote the same `image.json`, so the surviving sidecar describes whichever
message was written last. The message id is gone from the filename too.
The operator chose a clean re-download (2026-09-13) over relinking in place:
delete the broken files and their records, make gallery-dl forget it fetched
them, and backfill every Discord source again under the fixed naming.
## Why the archive is cleared for ALL of Discord
gallery-dl records a download as `discord{message_id}_{num}` (upstream
`DiscordExtractor.archive_fmt`, prefixed with the category). The broken files
lost their message ids, so there is no way to forget one source's entries and
not another's. Every Discord download made before the fix is broken, so
forgetting all of them is exactly right. Anything downloaded AFTER the fix still
exists on disk under its correct name, and gallery-dl's `skip` sees the file and
does not fetch it again.
## What it does not touch
Discord posts FC grouped itself (#388 E2) are built from Posts, and these files
never had one, so there is nothing grouped to unwind. Images outside a
`discord/None/` directory are never selected: the path filter requires both the
`None` directory and the `_None_` filename, the pair only the bug produced.
Operator-triggered only (Settings, preview first). Never on a beat.
"""
from __future__ import annotations
import logging
import sqlite3
from pathlib import Path
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from ..models import ImageRecord, Source
from .cleanup_service import delete_images
from .gallery_dl import archive_path
from .source_service import arm_backfill
log = logging.getLogger(__name__)
# `%` and `_` are LIKE wildcards, so the literal underscores around None are
# escaped. `________` is the eight-digit date prefix the old pattern wrote.
_BROKEN_PATH_LIKE = r"%/discord/None/________\_None\_%"
# Upstream keys asset downloads as `asset_{server_id}_{id}`. FC never fetches
# server assets, but excluding them keeps this to exactly the message
# attachments the bug mangled.
_ARCHIVE_SQL_MATCH = r"entry LIKE 'discord%' AND entry NOT LIKE 'discordasset\_%' ESCAPE '\'"
_COUNT_SQL = "SELECT COUNT(*) FROM archive WHERE " + _ARCHIVE_SQL_MATCH
_DELETE_SQL = "DELETE FROM archive WHERE " + _ARCHIVE_SQL_MATCH
def broken_directories(images_root: Path) -> list[Path]:
"""Every `<artist>/discord/None` directory. Artist folders that differ only by
case (`Conto` and `conto`) are separate directories and both are found."""
return sorted(d for d in Path(images_root).glob("*/discord/None") if d.is_dir())
def count_archive_entries(archive: Path) -> int:
return _archive(archive, delete=False)
def forget_archive_entries(archive: Path) -> int:
return _archive(archive, delete=True)
def _archive(archive: Path, *, delete: bool) -> int:
if not archive.is_file():
return 0
# A download running at the same moment holds this file briefly. Waiting
# 30s for its lock beats failing the repair over a transient contention.
conn = sqlite3.connect(str(archive), timeout=30)
try:
has_table = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='archive'"
).fetchone()
if not has_table:
return 0
if not delete:
return conn.execute(_COUNT_SQL).fetchone()[0]
cur = conn.execute(_DELETE_SQL)
conn.commit()
return cur.rowcount
finally:
conn.close()
def _sweep_directory(directory: Path) -> tuple[int, bool]:
"""Remove what the record deletes left behind: the collided sidecars, plus any
file that never became a record (a quarantined or rejected download). Then
the directory itself, if it is empty. Returns (files removed, dir removed)."""
removed = 0
for f in directory.iterdir():
if f.is_file():
try:
f.unlink()
removed += 1
except OSError as exc:
log.warning("discord repair: could not remove %s: %s", f, exc)
try:
directory.rmdir()
return removed, True
except OSError:
return removed, False
def repair_discord_downloads(
session: Session, *, images_root: Path, dry_run: bool,
) -> dict:
images_root = Path(images_root)
archive = archive_path(images_root)
broken = select(ImageRecord.id, ImageRecord.size_bytes).where(
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
)
rows = session.execute(broken).all()
image_ids = [r.id for r in rows]
directories = broken_directories(images_root)
sources = session.execute(
select(Source).where(Source.platform == "discord")
).scalars().all()
summary = {
"images": len(image_ids),
"bytes": sum(r.size_bytes or 0 for r in rows),
"directories": len(directories),
"sources": len(sources),
"enabled_sources": sum(1 for s in sources if s.enabled),
}
if dry_run:
summary["archive_entries"] = count_archive_entries(archive)
return summary
deleted = delete_images(session, image_ids=image_ids, images_root=images_root)
swept = 0
directories_removed = 0
for d in directories:
n, gone = _sweep_directory(d)
swept += n
directories_removed += int(gone)
# Only after the files are gone. Forgetting first and failing half way would
# leave gallery-dl free to re-fetch into a directory still full of the old
# copies.
forgotten = forget_archive_entries(archive)
for source in sources:
arm_backfill(source)
session.commit()
remaining = session.execute(
select(func.count(ImageRecord.id)).where(
ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\")
)
).scalar_one()
summary.update(
images_deleted=deleted["images_deleted"],
files_failed=deleted["files_failed"],
leftover_files_removed=swept,
directories_removed=directories_removed,
archive_entries=forgotten,
backfills_started=len(sources),
remaining=remaining,
)
log.info("discord repair applied: %s", summary)
return summary
+69 -3
View File
@@ -94,6 +94,58 @@ BACKFILL_CHUNK_SECONDS = 600
_DEFAULT_GDL_TIMEOUT_SECONDS = 870 _DEFAULT_GDL_TIMEOUT_SECONDS = 870
# --- Discord naming ---------------------------------------------------------
#
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
# for an attachment: `channel` is a plain STRING (the channel's name), the
# message is `message_id`, the attachment's position in it is `num`, and there
# is NO `id` key at all.
#
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
# "None", so every Discord download since the platform was added landed in a
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
# got a Post or a post date, and (b) collides: every `image.png` in a channel
# overwrote the same `image.json`, so the one sidecar that survived described
# whichever message happened to be written last.
#
# The fix names the sidecar EXACTLY like the media minus its extension, so
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
# these patterns against a sanitized copy of the real sidecar, so a key that
# does not exist fails CI instead of silently becoming "None".
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
DISCORD_DIRECTORY = ["{channel}"]
def sidecar_name_for(media_pattern: str) -> str | None:
"""The metadata filename pattern that names a sidecar exactly like its media.
Returns None for a pattern that does not end in `.{extension}`, since then
there is no media stem to mirror and the caller must fall back.
"""
suffix = ".{extension}"
if not media_pattern.endswith(suffix):
return None
return media_pattern[: -len(suffix)] + ".json"
def metadata_postprocessor(filename: str) -> dict:
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
def archive_path(images_root: Path) -> Path:
"""gallery-dl's download archive: the record of what it has already fetched.
One definition, because the Discord repair (services/discord_repair.py) has
to find the same file the downloader writes, without constructing a service
whose __init__ creates directories.
"""
return Path(images_root) / ".gallery-dl" / "archive.sqlite3"
@dataclass @dataclass
class SourceConfig: class SourceConfig:
"""Per-source overrides loaded from Source.config_overrides JSON. """Per-source overrides loaded from Source.config_overrides JSON.
@@ -376,8 +428,11 @@ class GalleryDLService:
}, },
"discord": { "discord": {
"content_types": ["all"], "content_types": ["all"],
"directory": ["{channel[name]}"], "directory": DISCORD_DIRECTORY,
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", "filename": DISCORD_FILENAME,
# Overrides the global `{filename}.json` sidecar for this extractor
# only — see the Discord naming note above.
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
"embeds": "all", "embeds": "all",
"stickers": True, "stickers": True,
"reactions": False, "reactions": False,
@@ -402,7 +457,7 @@ class GalleryDLService:
config = { config = {
"extractor": { "extractor": {
"base-directory": str(self.images_root), "base-directory": str(self.images_root),
"archive": str(self._config_dir / "archive.sqlite3"), "archive": str(archive_path(self.images_root)),
"skip": True, "skip": True,
"sleep": self._rate_limit, "sleep": self._rate_limit,
"sleep-request": max(0.5, self._rate_limit / 4), "sleep-request": max(0.5, self._rate_limit / 4),
@@ -497,6 +552,17 @@ class GalleryDLService:
if source_config.filename_pattern: if source_config.filename_pattern:
platform_section["filename"] = source_config.filename_pattern platform_section["filename"] = source_config.filename_pattern
# A platform that names its sidecar after its media must keep doing so
# under a per-source filename override, or the pairing breaks exactly the
# way Discord's did. No metadata wanted means no platform postprocessor
# either — the global list was already dropped above.
if "postprocessors" in platform_section:
mirrored = sidecar_name_for(platform_section.get("filename") or "")
if not source_config.save_metadata or mirrored is None:
platform_section.pop("postprocessors")
else:
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
platform_section["metadata"] = source_config.save_metadata platform_section["metadata"] = source_config.save_metadata
return config return config
+14 -7
View File
@@ -158,6 +158,19 @@ def _is_app_managed(key: str) -> bool:
BACKFILL_MAX_CHUNKS = 200 BACKFILL_MAX_CHUNKS = 200
def arm_backfill(source: Source) -> None:
"""Arm a fresh run-until-done backfill on `source` (plan #693). Mutation
only — the caller commits. Shared by `SourceService.start_backfill` and the
sync Discord repair, so both clear exactly the same resume state."""
co = dict(source.config_overrides or {})
co["_backfill_state"] = "running"
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
"_backfill_posts"):
co.pop(k, None)
source.config_overrides = co
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
class SourceService: class SourceService:
def __init__(self, session: AsyncSession): def __init__(self, session: AsyncSession):
self.session = session self.session = session
@@ -473,13 +486,7 @@ class SourceService:
)).scalar_one_or_none() )).scalar_one_or_none()
if source is None: if source is None:
raise LookupError(f"source id={source_id} not found") raise LookupError(f"source id={source_id} not found")
co = dict(source.config_overrides or {}) arm_backfill(source)
co["_backfill_state"] = "running"
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
"_backfill_posts"):
co.pop(k, None)
source.config_overrides = co
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
await self.session.commit() await self.session.commit()
return await self._row_to_record(source) return await self._row_to_record(source)
+22
View File
@@ -122,6 +122,28 @@ def prune_missing_file_records_task(self) -> dict:
return {"checked": checked, "missing": len(missing_ids), "deleted": deleted} return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
@celery.task(
name="backend.app.tasks.admin.repair_discord_downloads_task",
bind=True,
autoretry_for=(OperationalError, DBAPIError),
retry_backoff=15, retry_backoff_max=180, max_retries=1,
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
)
def repair_discord_downloads_task(self, dry_run: bool = True) -> dict:
"""Clean re-download of the Discord files broken by the `None` naming
(#3999). dry_run (the default) returns the projection; apply deletes the
broken images and their files, makes gallery-dl forget every Discord
download, and restarts every Discord source's backfill. Defaults to the SAFE
preview because the apply deletes files. Operator-triggered only."""
from ..services.discord_repair import repair_discord_downloads
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
return repair_discord_downloads(
session, images_root=IMAGES_ROOT, dry_run=dry_run,
)
@celery.task( @celery.task(
name="backend.app.tasks.admin.dedup_videos_task", name="backend.app.tasks.admin.dedup_videos_task",
bind=True, bind=True,
+73 -11
View File
@@ -1,5 +1,8 @@
<template> <template>
<v-card class="fc-post-card" variant="outlined"> <v-card
ref="cardEl" class="fc-post-card" variant="outlined"
:class="{ 'fc-post-card--wide': wide }"
>
<div class="fc-post-card__head"> <div class="fc-post-card__head">
<!-- Posts with no live subscription have source=null (alembic 0030); <!-- Posts with no live subscription have source=null (alembic 0030);
show a "filesystem import" affordance instead of a platform chip. --> show a "filesystem import" affordance instead of a platform chip. -->
@@ -58,7 +61,7 @@
</button> </button>
<div <div
v-if="rail.length || moreCount" class="fc-post-card__rail" v-if="rail.length || moreCount" class="fc-post-card__rail"
:style="{ '--fc-rail-cols': railCols }" :style="{ '--fc-rail-cols': railCols, '--fc-grid-cols': Math.min(2, railCols) }"
> >
<button <button
v-for="t in rail" :key="t.image_id" type="button" v-for="t in rail" :key="t.image_id" type="button"
@@ -221,20 +224,33 @@ const synthesisTitle = computed(() => {
}) })
const hero = computed(() => images.value[0]) const hero = computed(() => images.value[0])
// The thumbnail strip spans the hero's full width (CSS grid, equal columns),
// rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are // Filmstrip layout (milestone #407, option A). On a very wide window a card
// more images than fit, the last cell becomes a "+N" overflow tile so the // that just grew would give one post the whole screen, so a WIDE card instead
// count stays accurate. // pins the hero to a fixed height and moves the extra images into a 2-column
const RAIL_MAX = 5 // grid BESIDE it. Measured on the card rather than the viewport because the
// same card renders in the Latest feed, Browse, and the in-context view, each
// at a different width.
const WIDE_CARD_PX = 1100
const cardEl = ref(null)
const wide = ref(false)
// The narrow layout's strip spans the hero's full width (CSS grid, equal
// columns); the wide layout's grid is 2×2. Show up to that many cells; when
// there are more images than fit, the last cell becomes a "+N" overflow tile so
// the count stays accurate.
const RAIL_MAX_NARROW = 5
const RAIL_MAX_WIDE = 4
const serverMore = computed(() => props.post.thumbnails_more || 0) const serverMore = computed(() => props.post.thumbnails_more || 0)
const afterHero = computed(() => images.value.slice(1)) const afterHero = computed(() => images.value.slice(1))
const railMax = computed(() => (wide.value ? RAIL_MAX_WIDE : RAIL_MAX_NARROW))
const hasOverflow = computed( const hasOverflow = computed(
() => serverMore.value > 0 || afterHero.value.length > RAIL_MAX, () => serverMore.value > 0 || afterHero.value.length > railMax.value,
) )
const rail = computed(() => const rail = computed(() =>
hasOverflow.value hasOverflow.value
? afterHero.value.slice(0, RAIL_MAX - 1) ? afterHero.value.slice(0, railMax.value - 1)
: afterHero.value.slice(0, RAIL_MAX), : afterHero.value.slice(0, railMax.value),
) )
const visibleCount = computed(() => (images.value.length ? 1 + rail.value.length : 0)) const visibleCount = computed(() => (images.value.length ? 1 + rail.value.length : 0))
const moreCount = computed(() => { const moreCount = computed(() => {
@@ -349,8 +365,17 @@ function measureOverflow () {
} }
let ro = null let ro = null
let cardRo = null
onMounted(() => { onMounted(() => {
nextTick(measureOverflow) nextTick(measureOverflow)
const root = cardEl.value?.$el
if (typeof ResizeObserver !== 'undefined' && root) {
cardRo = new ResizeObserver((entries) => {
const w = entries[0]?.contentRect?.width ?? 0
wide.value = w >= WIDE_CARD_PX
})
cardRo.observe(root)
}
// Re-measure when the card resizes (the container-query clamp differs by // Re-measure when the card resizes (the container-query clamp differs by
// width). Guarded for happy-dom / older runtimes without ResizeObserver. // width). Guarded for happy-dom / older runtimes without ResizeObserver.
if (typeof ResizeObserver !== 'undefined' && descEl.value) { if (typeof ResizeObserver !== 'undefined' && descEl.value) {
@@ -358,7 +383,10 @@ onMounted(() => {
ro.observe(descEl.value) ro.observe(descEl.value)
} }
}) })
onBeforeUnmount(() => { if (ro) { ro.disconnect(); ro = null } }) onBeforeUnmount(() => {
if (ro) { ro.disconnect(); ro = null }
if (cardRo) { cardRo.disconnect(); cardRo = null }
})
async function toggleDesc () { async function toggleDesc () {
if (!descExpanded.value) { if (!descExpanded.value) {
@@ -495,6 +523,40 @@ function formatBytes (n) {
color: rgb(var(--v-theme-accent)); color: rgb(var(--v-theme-accent));
} }
/* --- Filmstrip layout (wide cards, #407 A) ---------------------------------
The hero has a HEIGHT, not a width: a wide card must not turn into a
full-screen post, so its height stays roughly a third of the viewport
whatever the window's width. The extra images sit beside it as square cells
whose size derives from that same height, so the grid always ends flush with
the hero's bottom edge. */
.fc-post-card--wide {
--fc-hero-h: clamp(260px, 34vh, 460px);
--fc-grid-gap: 8px;
--fc-cell: calc((var(--fc-hero-h) - var(--fc-grid-gap)) / 2);
}
.fc-post-card--wide .fc-post-card__body { flex-direction: row; gap: 24px; }
.fc-post-card--wide .fc-post-card__media {
flex: 0 0 auto;
display: flex;
gap: var(--fc-grid-gap);
}
.fc-post-card--wide .fc-post-card__hero {
width: auto;
height: var(--fc-hero-h);
}
.fc-post-card--wide .fc-post-card__rail {
margin-top: 0;
gap: var(--fc-grid-gap);
grid-template-columns: repeat(var(--fc-grid-cols, 2), var(--fc-cell));
grid-auto-rows: var(--fc-cell);
}
.fc-post-card--wide .fc-post-card__rail-cell,
.fc-post-card--wide .fc-post-card__rail-more { height: 100%; }
.fc-post-card--wide .fc-post-card__text { flex: 1 1 0; min-width: 0; }
/* Text is secondary here — long reads happen in the expanded view — so the
clamp keeps the text column no taller than the images beside it. */
.fc-post-card--wide .fc-post-card__desc--clamped { -webkit-line-clamp: 4; }
.fc-post-card__title { .fc-post-card__title {
font-family: 'Fraunces', Georgia, serif; font-family: 'Fraunces', Georgia, serif;
font-size: 18px; font-weight: 700; font-size: 18px; font-weight: 700;
@@ -0,0 +1,118 @@
<template>
<!-- #3999: Discord downloads made before the naming fix landed in a `None`
folder with no post and a download-time date. This re-downloads them
cleanly. Preview first; the apply deletes files. -->
<MaintenanceTile
icon="mdi-download-off-outline"
title="Repair Discord downloads"
blurb="Re-download Discord images that were saved with no post and the wrong date."
destructive
:open="applying || previewing"
>
<p class="text-body-2 mb-3">
Before the naming fix, every Discord download was saved into a folder
called <code>None</code>. Those images never got a post, so they show the
time Curator downloaded them rather than when they were posted.
<strong>Apply</strong> deletes those images, clears Discord from the
download history, and starts a fresh backfill of every Discord source so
they come back with their posts and dates. Tags you added by hand to those
images are lost. Nothing outside a Discord <code>None</code> folder is
touched.
</p>
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
<v-btn
color="primary" variant="tonal" rounded="pill"
:loading="previewing" :disabled="applying" @click="preview"
>
<v-icon start>mdi-magnify</v-icon> Preview
</v-btn>
<v-btn
color="error" rounded="pill"
:loading="applying"
:disabled="previewing || !canApply"
@click="confirmOpen = true"
>
<v-icon start>mdi-download-off-outline</v-icon> Apply
</v-btn>
</div>
<v-alert
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
density="comfortable"
>
<span v-if="applied">
Deleted {{ summary.images_deleted }} image(s) and cleared
{{ summary.archive_entries }} Discord download record(s). Backfills
started on {{ summary.backfills_started }} Discord source(s).
</span>
<span v-else-if="hasWork">
{{ summary.images }} broken image(s) ({{ humanBytes(summary.bytes) }})
across {{ summary.directories }} folder(s), and
{{ summary.archive_entries }} Discord download record(s) to clear.
{{ summary.sources }} Discord source(s) will backfill again.
</span>
<span v-else>Nothing to repair no broken Discord downloads found.</span>
<div v-if="applied && summary.files_failed" class="mt-1 text-caption">
{{ summary.files_failed }} file(s) could not be removed see the worker log.
</div>
<div v-if="applied && summary.remaining" class="mt-1 text-caption">
{{ summary.remaining }} broken image(s) are still present. Run it again.
</div>
</v-alert>
<QueueStatusBar queue="maintenance_long" queue-label="Maintenance" />
<v-dialog v-model="confirmOpen" max-width="440">
<v-card>
<v-card-title>Repair Discord downloads?</v-card-title>
<v-card-text class="text-body-2">
This permanently deletes <strong>{{ summary?.images ?? 0 }}</strong>
Discord image(s) ({{ humanBytes(summary?.bytes) }}) and re-downloads
them from Discord with their posts and dates. Any tags you added to
those images by hand will not come back.
</v-card-text>
<v-card-actions>
<v-spacer />
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
<v-btn color="error" @click="apply">Repair</v-btn>
</v-card-actions>
</v-card>
</v-dialog>
</MaintenanceTile>
</template>
<script setup>
import { computed, ref } from 'vue'
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
import { humanBytes } from '../../utils/bytes.js'
import MaintenanceTile from '../common/MaintenanceTile.vue'
import QueueStatusBar from './QueueStatusBar.vue'
const confirmOpen = ref(false)
const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
endpoint: '/api/admin/maintenance/repair-discord-downloads',
storageKey: 'fc.maint.repairDiscordDownloads',
appliedToast: 'Discord downloads cleared — backfills started',
})
// The archive count matters too: files could already be gone while gallery-dl
// still believes it has them, which would stop the backfill re-fetching.
const hasWork = computed(
() => !!summary.value && (summary.value.images > 0 || summary.value.archive_entries > 0),
)
const canApply = computed(() => hasWork.value && !applied.value)
const summaryType = computed(() => {
if (applied.value) return 'success'
return hasWork.value ? 'info' : 'success'
})
// The confirm dialog gates the destructive apply; close it, then run.
function apply () {
confirmOpen.value = false
applyTask()
}
</script>
@@ -15,6 +15,7 @@
<ImportFiltersForm /> <ImportFiltersForm />
<TranslationCard /> <TranslationCard />
<DiscordGroupingCard /> <DiscordGroupingCard />
<DiscordRepairCard />
<PostAssociationsCard /> <PostAssociationsCard />
<MembershipRosterCard /> <MembershipRosterCard />
<MembershipSuggestionsCard /> <MembershipSuggestionsCard />
@@ -85,6 +86,7 @@ import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
import CropProposersCard from './CropProposersCard.vue' import CropProposersCard from './CropProposersCard.vue'
import HeadsCard from './HeadsCard.vue' import HeadsCard from './HeadsCard.vue'
import DiscordGroupingCard from './DiscordGroupingCard.vue' import DiscordGroupingCard from './DiscordGroupingCard.vue'
import DiscordRepairCard from './DiscordRepairCard.vue'
import MembershipRosterCard from './MembershipRosterCard.vue' import MembershipRosterCard from './MembershipRosterCard.vue'
import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue' import MembershipSuggestionsCard from './MembershipSuggestionsCard.vue'
import PostAssociationsCard from './PostAssociationsCard.vue' import PostAssociationsCard from './PostAssociationsCard.vue'
+148 -11
View File
@@ -1,5 +1,8 @@
<template> <template>
<v-container class="pt-2 pb-6" max-width="900"> <!-- Width is set in CSS, not with `max-width` here: below 1600px it is
today's 900px column, and above it the feed widens and gains the rail
and day gutter (milestone #407). -->
<v-container fluid class="pt-2 pb-6 fc-posts">
<!-- In-context view: deep-linked to one post, with bidirectional infinite <!-- In-context view: deep-linked to one post, with bidirectional infinite
scroll — newer posts load above, older posts below. --> scroll — newer posts load above, older posts below. -->
<template v-if="postIdFilter != null"> <template v-if="postIdFilter != null">
@@ -48,15 +51,19 @@
</template> </template>
<!-- Normal feed --> <!-- Normal feed -->
<template v-else> <div v-else class="fc-posts__layout">
<FeedStatusRibbon v-if="statusRibbon" /> <!-- On a wide window this is a sticky left rail (#407 E); below the
breakpoint it lays out exactly as the old inline header did. -->
<PostsFilterBar <aside class="fc-posts__rail">
:artist-id="artistFilter" <PostsFilterBar
:platform="platformFilter" :artist-id="artistFilter"
@update:filters="onFilters" :platform="platformFilter"
/> @update:filters="onFilters"
/>
<FeedStatusRibbon v-if="statusRibbon" />
</aside>
<div class="fc-posts__main">
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3"> <v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
{{ String(store.error) }} {{ String(store.error) }}
</v-alert> </v-alert>
@@ -74,14 +81,28 @@
</div> </div>
<div v-else> <div v-else>
<PostCard v-for="p in store.items" :key="p.id" :post="p" /> <!-- Day groups (#407 D). The heading sits above its posts on a narrow
window and in a sticky left gutter on a wide one. -->
<section v-for="d in days" :key="d.key" class="fc-posts__day">
<header class="fc-posts__day-head">
<span class="fc-posts__day-label">{{ d.label }}</span>
<span class="fc-posts__day-count">
{{ d.posts.length }} post{{ d.posts.length === 1 ? '' : 's' }}
· {{ d.artistCount }} artist{{ d.artistCount === 1 ? '' : 's' }}
</span>
</header>
<div class="fc-posts__day-posts">
<PostCard v-for="p in d.posts" :key="p.id" :post="p" />
</div>
</section>
<div ref="sentinel" class="fc-posts__sentinel"> <div ref="sentinel" class="fc-posts__sentinel">
<v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" /> <v-progress-circular v-if="store.loading" indeterminate color="accent" size="24" />
<span v-else-if="store.done" class="fc-posts__end">End of stream</span> <span v-else-if="store.done" class="fc-posts__end">End of stream</span>
</div> </div>
</div> </div>
</template> </div>
</div>
</v-container> </v-container>
</template> </template>
@@ -121,6 +142,48 @@ const hasActiveFilter = computed(() =>
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
) )
// --- day groups (#407 D) ---
// CONSECUTIVE runs, not a bucket per date. The feed's sort key includes
// `resurfaced_at` (a Discord grouping that grew moves back to the top), which
// the payload does not carry, so a resurfaced post can sit above newer ones.
// Bucketing by date would pull it out of order; a run gives it its own heading
// where it actually appears. Counts cover what has LOADED, and grow as the
// infinite scroll fetches more of the same day.
function dayKey (iso) {
const d = new Date(iso)
return `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`
}
function dayLabel (iso) {
const d = new Date(iso)
const today = new Date()
const startOf = (x) => new Date(x.getFullYear(), x.getMonth(), x.getDate()).getTime()
const days = Math.round((startOf(today) - startOf(d)) / 86400000)
if (days === 0) return 'Today'
if (days === 1) return 'Yesterday'
if (days > 1 && days < 7) return d.toLocaleDateString(undefined, { weekday: 'long' })
const sameYear = d.getFullYear() === today.getFullYear()
return d.toLocaleDateString(undefined, {
month: 'short', day: 'numeric', ...(sameYear ? {} : { year: 'numeric' }),
})
}
const days = computed(() => {
const groups = []
for (const p of store.items) {
const iso = p.post_date || p.downloaded_at
const key = dayKey(iso)
let g = groups[groups.length - 1]
if (!g || g.dayKey !== key) {
// Suffix with the run index so a day that appears twice (see above)
// still has a unique v-for key.
g = { key: `${key}#${groups.length}`, dayKey: key, label: dayLabel(iso), posts: [], artists: new Set() }
groups.push(g)
}
g.posts.push(p)
if (p.artist?.id != null) g.artists.add(p.artist.id)
}
return groups.map((g) => ({ ...g, artistCount: g.artists.size }))
})
// Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and // Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and
// any active artist/platform scope) intact instead of resetting the surface. // any active artist/platform scope) intact instead of resetting the surface.
const allPostsTarget = computed(() => { const allPostsTarget = computed(() => {
@@ -232,6 +295,80 @@ onUnmounted(() => { teardownFeed(); teardownAround() })
</script> </script>
<style scoped> <style scoped>
/* Below the breakpoint: today's layout exactly — a 900px column with the
filters and ribbon inline above the feed. */
.fc-posts { max-width: 900px; }
.fc-posts__day-head {
display: flex;
align-items: baseline;
gap: 10px;
padding: 4px 0 8px;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-posts__day-label {
font-family: 'Fraunces', Georgia, serif;
font-weight: 600;
font-size: 1rem;
color: rgb(var(--v-theme-accent));
}
.fc-posts__day-count { font-size: 0.78rem; }
/* Wide window (#407 D + E). The rail holds filters and status; each day's
heading moves into a sticky gutter beside its posts; the column widens and
the cards switch to their filmstrip layout on their own (PostCard measures
itself). 1600px is where a 280px rail and a 150px gutter still leave a card
wide enough to be worth the change. */
@media (min-width: 1600px) {
.fc-posts { max-width: 2360px; }
.fc-posts__layout {
display: grid;
grid-template-columns: 280px minmax(0, 1fr);
gap: 40px;
align-items: start;
}
.fc-posts__rail {
position: sticky;
top: calc(var(--fc-nav-h, 64px) + 16px);
display: flex;
flex-direction: column;
gap: 16px;
}
.fc-posts__rail :deep(.fc-posts-filters) {
flex-direction: column;
align-items: stretch;
padding-bottom: 0;
}
.fc-posts__rail :deep(.fc-posts-filters__artist),
.fc-posts__rail :deep(.fc-posts-filters__platform) {
flex: none;
width: 100%;
min-width: 0;
max-width: none;
}
.fc-posts__rail :deep(.fc-ribbon) {
flex-direction: column;
align-items: flex-start;
gap: 8px;
}
.fc-posts__main { max-width: 1900px; }
.fc-posts__day {
display: grid;
grid-template-columns: 150px minmax(0, 1fr);
gap: 0 24px;
}
.fc-posts__day-head {
position: sticky;
top: calc(var(--fc-nav-h, 64px) + 16px);
align-self: start;
flex-direction: column;
align-items: flex-end;
gap: 2px;
padding-top: 12px;
text-align: right;
}
.fc-posts__day-label { font-size: 1.1rem; }
}
.fc-posts__loading, .fc-posts__loading,
.fc-posts__empty { .fc-posts__empty {
display: flex; display: flex;
+28
View File
@@ -0,0 +1,28 @@
{
"author": "example-artist",
"author_files": [],
"author_id": "100000000000000001",
"category": "discord",
"channel": "nsfw-drops",
"channel_id": "200000000000000002",
"channel_topic": "",
"channel_type": 0,
"date": "2024-07-16 16:43:23",
"extension": "png",
"filename": "image",
"files": [],
"is_thread": false,
"message": "",
"message_id": "300000000000000003",
"num": 1,
"owner_id": "100000000000000001",
"parent": "",
"parent_id": "",
"parent_type": 0,
"server": "Example Server",
"server_files": [],
"server_id": "400000000000000004",
"subcategory": "channel",
"type": "attachment",
"url": "https://cdn.discordapp.com/attachments/200000000000000002/500000000000000005/image.png"
}
+170
View File
@@ -0,0 +1,170 @@
"""The Discord repair (#3999): forget the broken downloads, delete them, backfill again.
The danger in a repair like this is reach. It deletes files and clears
gallery-dl's memory of what it fetched, so most of these tests pin what it must
NOT touch: other platforms' archive entries, Discord server assets, images
outside a `discord/None/` folder, and a correctly named Discord file that
happens to share a folder name.
"""
import sqlite3
import pytest
from sqlalchemy import func, select
from backend.app.models import Artist, ImageRecord, Source
from backend.app.services import discord_repair
from backend.app.services.gallery_dl import archive_path
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
def _archive(tmp_path, entries):
path = archive_path(tmp_path)
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path)
# The table gallery-dl itself creates (gallery_dl/archive.py).
conn.execute("CREATE TABLE archive (entry TEXT PRIMARY KEY) WITHOUT ROWID")
conn.executemany("INSERT INTO archive (entry) VALUES (?)", [(e,) for e in entries])
conn.commit()
conn.close()
return path
def _entries(path):
conn = sqlite3.connect(path)
try:
return sorted(r[0] for r in conn.execute("SELECT entry FROM archive"))
finally:
conn.close()
# --- the archive ------------------------------------------------------------
def test_only_discord_message_entries_are_forgotten(tmp_path):
path = _archive(tmp_path, [
"discord300000000000000003_1",
"discord300000000000000004_2",
"discordasset_400000000000000004_9",
"hentaifoundry12345",
])
assert discord_repair.count_archive_entries(path) == 2
assert discord_repair.forget_archive_entries(path) == 2
assert _entries(path) == ["discordasset_400000000000000004_9", "hentaifoundry12345"]
def test_counting_changes_nothing(tmp_path):
path = _archive(tmp_path, ["discord1_1", "discord2_1"])
discord_repair.count_archive_entries(path)
assert _entries(path) == ["discord1_1", "discord2_1"]
def test_a_missing_archive_or_table_is_zero_not_an_error(tmp_path):
assert discord_repair.count_archive_entries(tmp_path / "nope.sqlite3") == 0
empty = tmp_path / "empty.sqlite3"
sqlite3.connect(empty).close()
assert discord_repair.forget_archive_entries(empty) == 0
# --- the folders ------------------------------------------------------------
def test_broken_directories_finds_every_artist_folder_including_case_twins(tmp_path):
for artist in ("Conto", "conto", "knuxy"):
(tmp_path / artist / "discord" / "None").mkdir(parents=True)
(tmp_path / "conto" / "discord" / "general").mkdir(parents=True)
(tmp_path / "conto" / "patreon" / "None").mkdir(parents=True)
found = [
p.relative_to(tmp_path).as_posix()
for p in discord_repair.broken_directories(tmp_path)
]
assert found == ["Conto/discord/None", "conto/discord/None", "knuxy/discord/None"]
def test_sweep_removes_leftover_sidecars_and_the_empty_folder(tmp_path):
d = tmp_path / "conto" / "discord" / "None"
d.mkdir(parents=True)
(d / "image.json").write_text("{}")
(d / "20240716_None_rejected.png").write_bytes(b"x")
removed, gone = discord_repair._sweep_directory(d)
assert (removed, gone) == (2, True)
assert not d.exists()
# --- the whole repair, against the database -----------------------------------
def _image(db_sync, artist, path, sha):
img = ImageRecord(
artist_id=artist.id, path=str(path), sha256=sha * 64, size_bytes=100,
mime="image/png", origin="downloaded",
)
db_sync.add(img)
return img
@pytest.mark.integration
def test_repair_deletes_only_broken_discord_images_and_rearms_discord_backfills(
db_sync, tmp_path,
):
artist = Artist(name="Conto", slug="conto")
db_sync.add(artist)
db_sync.flush()
discord = Source(
artist_id=artist.id, platform="discord", url="https://discord.com/channels/1/2",
enabled=True, config_overrides={"_backfill_state": "complete", "_backfill_cursor": "x"},
)
patreon = Source(
artist_id=artist.id, platform="patreon", url="https://www.patreon.com/conto",
enabled=True, config_overrides={},
)
db_sync.add_all([discord, patreon])
broken_dir = tmp_path / "conto" / "discord" / "None"
good_dir = tmp_path / "conto" / "discord" / "general"
patreon_dir = tmp_path / "conto" / "patreon" / "None"
for d in (broken_dir, good_dir, patreon_dir):
d.mkdir(parents=True)
broken = broken_dir / "20240716_None_image.png"
good = good_dir / "20240716_300000000000000003_01_image.png"
# A `None` folder under another platform, and a `_None_` name outside one.
# Neither is the bug's pair, so neither may be touched.
elsewhere = patreon_dir / "20240716_None_image.png"
for f in (broken, good, elsewhere):
f.write_bytes(b"x")
(broken_dir / "image.json").write_text("{}")
_image(db_sync, artist, broken, "a")
_image(db_sync, artist, good, "b")
_image(db_sync, artist, elsewhere, "c")
db_sync.commit()
_archive(tmp_path, ["discord300000000000000003_1", "hentaifoundry1"])
preview = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=True)
assert preview["images"] == 1
assert preview["archive_entries"] == 1
assert broken.exists()
result = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=False)
assert result["images_deleted"] == 1
assert result["remaining"] == 0
assert result["backfills_started"] == 1
assert not broken.exists() and not broken_dir.exists()
assert good.exists() and elsewhere.exists()
assert _entries(archive_path(tmp_path)) == ["hentaifoundry1"]
kept = db_sync.execute(select(func.count(ImageRecord.id))).scalar_one()
assert kept == 2
rows = {
platform: (runs, overrides)
for platform, runs, overrides in db_sync.execute(
select(Source.platform, Source.backfill_runs_remaining, Source.config_overrides)
).all()
}
runs, overrides = rows["discord"]
assert runs == BACKFILL_MAX_CHUNKS
assert overrides["_backfill_state"] == "running"
assert "_backfill_cursor" not in overrides
# Only Discord sources are re-armed.
assert rows["patreon"][1] == {}
+117
View File
@@ -0,0 +1,117 @@
"""gallery-dl naming patterns, rendered against the metadata gallery-dl really emits.
gallery-dl does not fail on a format field that names a key the extractor never
sets. It renders "None" and carries on. That is how every Discord download
landed in a `None/` directory as `<date>_None_<name>`: the patterns asked for
`{channel[name]}` and `{id}`, and the real metadata has a string `channel` and
`message_id`, with no `id` at all. The mismatch also broke sidecar pairing, so
no Discord file ever got a Post or a post date.
The fixture keeps the key set and value TYPES of a real attachment sidecar from
the operator's instance (2026-09-13), with every value invented. Rendering
through Python's own formatter raises on a missing key or a subscript into a
string, which is the loud failure gallery-dl does not give.
"""
import json
import string
from datetime import datetime
from pathlib import Path
import pytest
from backend.app.services.gallery_dl import (
DISCORD_DIRECTORY,
DISCORD_FILENAME,
GalleryDLService,
SourceConfig,
sidecar_name_for,
)
from backend.app.utils.sidecar import find_sidecar
_FIXTURE = Path(__file__).parent / "fixtures" / "discord_attachment_sidecar.json"
@pytest.fixture
def kwdict():
data = json.loads(_FIXTURE.read_text())
# gallery-dl hands the formatter a datetime; the JSON sidecar stores it as text.
data["date"] = datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
return data
def _render(pattern, kwdict):
return string.Formatter().vformat(pattern, (), kwdict)
def test_the_discord_filename_renders_from_real_keys(kwdict):
name = _render(DISCORD_FILENAME, kwdict)
assert "None" not in name
assert name == "20240716_300000000000000003_01_image.png"
def test_the_discord_directory_renders_from_real_keys(kwdict):
assert [_render(p, kwdict) for p in DISCORD_DIRECTORY] == ["nsfw-drops"]
@pytest.mark.parametrize(
"broken", ["{channel[name]}", "{date:%Y%m%d}_{id}_{filename}.{extension}"],
)
def test_the_patterns_that_shipped_would_have_failed_here(kwdict, broken):
"""Positive control: the two patterns that produced `None/..._None_...`
must fail this renderer, or the tests above prove nothing."""
with pytest.raises((KeyError, TypeError)):
_render(broken, kwdict)
def test_the_sidecar_is_named_exactly_like_its_media(kwdict, tmp_path):
"""So `find_sidecar` pairs them on its first candidate, and two attachments
that share an original name (`image.png`) can never share a sidecar."""
media = tmp_path / _render(DISCORD_FILENAME, kwdict)
sidecar = tmp_path / _render(sidecar_name_for(DISCORD_FILENAME), kwdict)
media.write_bytes(b"x")
sidecar.write_text("{}")
assert find_sidecar(media) == sidecar
def test_two_attachments_with_the_same_original_name_get_distinct_sidecars(kwdict):
other = {**kwdict, "message_id": "300000000000000099"}
pattern = sidecar_name_for(DISCORD_FILENAME)
assert _render(pattern, kwdict) != _render(pattern, other)
def test_sidecar_name_for_needs_an_extension_suffix():
assert sidecar_name_for("{a}_{b}.{extension}") == "{a}_{b}.json"
assert sidecar_name_for("{a}_{b}") is None
# --- the config gallery-dl is actually given -----------------------------------
@pytest.fixture
def gdl(tmp_path):
return GalleryDLService(images_root=tmp_path / "images", validate_files=False)
def _discord_section(gdl, **overrides):
cfg = gdl._build_config_for_source(
platform="discord", source_config=SourceConfig(**overrides), artist_slug="a",
)
return cfg["extractor"]["discord"]
def test_discord_config_carries_the_mirrored_sidecar(gdl):
section = _discord_section(gdl)
assert section["filename"] == DISCORD_FILENAME
assert section["directory"] == DISCORD_DIRECTORY
assert section["postprocessors"][0]["filename"] == sidecar_name_for(DISCORD_FILENAME)
def test_a_filename_override_keeps_the_sidecar_mirrored(gdl):
section = _discord_section(gdl, filename_pattern="{message_id}_{num}.{extension}")
assert section["postprocessors"][0]["filename"] == "{message_id}_{num}.json"
def test_no_metadata_means_no_discord_postprocessor(gdl):
section = _discord_section(gdl, save_metadata=False)
assert "postprocessors" not in section