Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f2f6b6d25e | |||
| 7a5a71471e | |||
| 38a45baad5 | |||
| c9ddcd0f60 | |||
| 95bc761a69 | |||
| 6acf273267 | |||
| 0822240fde | |||
| f5efbea053 | |||
| f653c26680 | |||
| 27f7f3fd01 | |||
| 505dca1b4d | |||
| 061dc9e605 | |||
| 7d8b9c3d90 | |||
| 8f25b27315 | |||
| d6eba8e4bd | |||
| 6f68bf5fa7 | |||
| ec44c653fe | |||
| c5bf564f53 | |||
| 8188985029 | |||
| 7f26217166 |
+5
-2
@@ -16,12 +16,15 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
# System deps: ffmpeg (transcode + thumbnails, used in FC-2), unar (archives, FC-2),
|
||||
# libpq for psycopg, image libs.
|
||||
# System deps: ffmpeg (transcode + thumbnails, FC-2), unar (archives, FC-2),
|
||||
# libpq for psycopg, postgresql-client + zstd for FC-5 backup/restore
|
||||
# (pg_dump + tar --zstd), image libs.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
unar \
|
||||
libpq5 \
|
||||
postgresql-client \
|
||||
zstd \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
|
||||
@@ -23,6 +23,12 @@ def create_app() -> Quart:
|
||||
|
||||
app = Quart(__name__)
|
||||
app.secret_key = cfg.secret_key
|
||||
# FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of
|
||||
# thousands of image_tag_associations). Werkzeug's default form
|
||||
# memory cap is 500KB; raise both ceilings so the multipart upload
|
||||
# for /api/migrate/ir_ingest doesn't 413.
|
||||
app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GB
|
||||
app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024 # 1 GB
|
||||
|
||||
for bp in all_blueprints():
|
||||
app.register_blueprint(bp)
|
||||
|
||||
@@ -21,10 +21,10 @@ migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
||||
|
||||
_VALID_KINDS = frozenset({
|
||||
"backup", "gs_ingest", "ir_ingest", "tag_apply",
|
||||
"ml_queue", "verify", "rollback",
|
||||
"ml_queue", "verify", "rollback", "cleanup",
|
||||
})
|
||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback"})
|
||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
|
||||
@@ -6,7 +6,7 @@ from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||
from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||
|
||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
||||
|
||||
@@ -118,6 +118,9 @@ async def system_stats():
|
||||
storage_bytes = (
|
||||
(await session.execute(select(func.coalesce(func.sum(ImageRecord.size_bytes), 0)))).scalar_one()
|
||||
)
|
||||
subscription_count = (await session.execute(
|
||||
select(func.count(Artist.id)).where(Artist.is_subscription.is_(True))
|
||||
)).scalar_one()
|
||||
|
||||
# Task counts grouped by status
|
||||
status_rows = (
|
||||
@@ -138,11 +141,23 @@ async def system_stats():
|
||||
).all()
|
||||
integrity_counts = {row[0]: row[1] for row in integrity_rows}
|
||||
|
||||
# Active batch (most recent running)
|
||||
# Active batch = running batch that still has outstanding work.
|
||||
# Plain "most recent running" picks a freshly-created scan that
|
||||
# enqueued zero new files and hides the older batch that's
|
||||
# actually being processed; the EXISTS clause filters those
|
||||
# empty batches out.
|
||||
active_batch_row = (
|
||||
await session.execute(
|
||||
select(ImportBatch)
|
||||
.where(ImportBatch.status == "running")
|
||||
.where(
|
||||
ImportBatch.status == "running",
|
||||
select(ImportTask.id)
|
||||
.where(
|
||||
ImportTask.batch_id == ImportBatch.id,
|
||||
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||
)
|
||||
.exists(),
|
||||
)
|
||||
.order_by(ImportBatch.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -163,6 +178,7 @@ async def system_stats():
|
||||
"total_images": total_images,
|
||||
"total_tags": total_tags,
|
||||
"storage_bytes": storage_bytes,
|
||||
"subscription_count": int(subscription_count),
|
||||
"tasks": {
|
||||
"pending": status_counts.get("pending", 0),
|
||||
"queued": status_counts.get("queued", 0),
|
||||
|
||||
+24
-2
@@ -1,14 +1,36 @@
|
||||
"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback."""
|
||||
"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback,
|
||||
and the on-disk image library + thumbnails from /images.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, send_from_directory
|
||||
from quart import Blueprint, abort, send_from_directory
|
||||
|
||||
FRONTEND_DIST = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist"
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
frontend_bp = Blueprint("frontend", __name__)
|
||||
|
||||
|
||||
@frontend_bp.route("/images/<path:subpath>")
|
||||
async def serve_image(subpath: str):
|
||||
"""Serve a file from the /images volume (originals + thumbnails).
|
||||
|
||||
Without this route the SPA catch-all below would swallow image
|
||||
requests and return index.html, leaving the browser to render the
|
||||
aspect-ratio-shaped grey placeholder.
|
||||
"""
|
||||
target = (IMAGES_ROOT / subpath).resolve()
|
||||
# Defend against path-traversal: refuse anything that escapes /images.
|
||||
try:
|
||||
target.relative_to(IMAGES_ROOT)
|
||||
except ValueError:
|
||||
abort(404)
|
||||
if not target.is_file():
|
||||
abort(404)
|
||||
return await send_from_directory(IMAGES_ROOT, subpath)
|
||||
|
||||
|
||||
@frontend_bp.route("/")
|
||||
@frontend_bp.route("/<path:subpath>")
|
||||
async def serve_spa(subpath: str = ""):
|
||||
|
||||
@@ -16,6 +16,19 @@ from typing import Any
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
|
||||
|
||||
def _libpq_url(sa_url: str) -> str:
|
||||
"""Strip SQLAlchemy driver suffix so pg_dump/psql accept the URL.
|
||||
|
||||
SQLAlchemy uses URLs like `postgresql+psycopg://...` or
|
||||
`postgresql+asyncpg://...`. libpq tools (pg_dump, psql) only know
|
||||
the plain `postgresql://` scheme.
|
||||
"""
|
||||
for driver in ("postgresql+psycopg", "postgresql+asyncpg", "postgresql+psycopg2"):
|
||||
if sa_url.startswith(driver + "://"):
|
||||
return "postgresql://" + sa_url[len(driver) + 3:]
|
||||
return sa_url
|
||||
|
||||
|
||||
def _backups_dir(images_root: Path | None = None) -> Path:
|
||||
# Overridable for tests via monkeypatch.
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
@@ -49,7 +62,7 @@ def create_backup(
|
||||
manifest_path = out_dir / f"fc_{ts}.json"
|
||||
|
||||
_run_subprocess(
|
||||
["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), db_url],
|
||||
["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), _libpq_url(db_url)],
|
||||
_test_ts=ts,
|
||||
)
|
||||
_run_subprocess(
|
||||
@@ -104,7 +117,7 @@ def restore_backup(
|
||||
tar_path = Path(manifest["tar_path"])
|
||||
|
||||
_run_subprocess(
|
||||
["psql", "-d", db_url, "-f", str(sql_path)],
|
||||
["psql", "-d", _libpq_url(db_url), "-f", str(sql_path)],
|
||||
)
|
||||
|
||||
# Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Targeted cleanup migrator: delete every image attributed to one Artist.
|
||||
|
||||
Built for the IR-migration rescue case where the filesystem scan derived
|
||||
a bogus 'imagerepo' artist from a mismatched bind-mount layout. Every
|
||||
image attributed to that artist (40k+ rows) needs to be removed — DB
|
||||
rows, original files under `/images/<bucket>/...`, and thumbnails under
|
||||
`/images/thumbs/...` — before the operator remounts and re-scans.
|
||||
|
||||
CASCADE handles image_tag, image_provenance, series_page, and
|
||||
tag_suggestion_rejection child rows; import_task.result_image_id is
|
||||
SET NULL by FK. We also delete ImportTask rows whose source_path starts
|
||||
with the (still-existing) IR scan prefix so the next scan isn't fooled
|
||||
by them.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Artist, ImageRecord, ImportBatch, ImportTask
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BATCH_SIZE = 500
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
||||
}
|
||||
|
||||
|
||||
def _thumb_path(images_root: Path, sha256_hex: str) -> tuple[Path, Path]:
|
||||
"""Return both possible thumbnail paths (.jpg and .png). We try both
|
||||
because the extension is chosen at generate-time based on the source
|
||||
image's mode (alpha → .png, otherwise → .jpg)."""
|
||||
bucket = sha256_hex[:3]
|
||||
base = images_root / "thumbs" / bucket / sha256_hex
|
||||
return base.with_suffix(".jpg"), base.with_suffix(".png")
|
||||
|
||||
|
||||
def _delete_file(path: Path) -> bool:
|
||||
"""Best-effort unlink; True if the file was actually removed."""
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError as exc:
|
||||
log.warning("cleanup: failed to unlink %s: %s", path, exc)
|
||||
return False
|
||||
|
||||
|
||||
async def cleanup_artist_async(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
slug: str,
|
||||
images_root: Path | None = None,
|
||||
dry_run: bool = False,
|
||||
source_path_prefix: str | None = None,
|
||||
) -> dict:
|
||||
"""Delete every image attributed to the Artist with this slug,
|
||||
along with the artist row itself and any associated import tasks.
|
||||
|
||||
Args:
|
||||
slug: artist.slug to target (e.g. 'imagerepo').
|
||||
images_root: defaults to /images.
|
||||
dry_run: skip filesystem + DB writes; still walk rows for counts.
|
||||
source_path_prefix: if set, ImportTask rows whose source_path
|
||||
starts with this string are deleted too (use the IR scan
|
||||
mount prefix, e.g. '/import/imagerepo').
|
||||
"""
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
|
||||
artist = (await db.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
raise ValueError(f"no Artist with slug={slug!r}")
|
||||
|
||||
artist_id = artist.id
|
||||
artist_name = artist.name
|
||||
|
||||
total_images = (await db.execute(
|
||||
select(func.count(ImageRecord.id)).where(ImageRecord.artist_id == artist_id)
|
||||
)).scalar_one()
|
||||
|
||||
counts = _zero_counts()
|
||||
files_deleted = 0
|
||||
thumbs_deleted = 0
|
||||
images_deleted = 0
|
||||
|
||||
# Batched delete loop. CASCADE handles image_tag, image_provenance,
|
||||
# series_page, tag_suggestion_rejection. import_task.result_image_id
|
||||
# is SET NULL by FK.
|
||||
while True:
|
||||
rows = (await db.execute(
|
||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
||||
.where(ImageRecord.artist_id == artist_id)
|
||||
.limit(_BATCH_SIZE)
|
||||
)).all()
|
||||
if not rows:
|
||||
break
|
||||
|
||||
ids = [r.id for r in rows]
|
||||
counts["rows_processed"] += len(ids)
|
||||
|
||||
if not dry_run:
|
||||
for r in rows:
|
||||
if r.path:
|
||||
if _delete_file(Path(r.path)):
|
||||
files_deleted += 1
|
||||
if r.sha256:
|
||||
jpg, png = _thumb_path(root, r.sha256)
|
||||
if _delete_file(jpg):
|
||||
thumbs_deleted += 1
|
||||
if _delete_file(png):
|
||||
thumbs_deleted += 1
|
||||
|
||||
await db.execute(
|
||||
delete(ImageRecord).where(ImageRecord.id.in_(ids))
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
images_deleted += len(ids)
|
||||
|
||||
if dry_run:
|
||||
# Nothing was actually deleted from the DB; bail after one
|
||||
# pass so we don't loop forever.
|
||||
break
|
||||
|
||||
import_tasks_deleted = 0
|
||||
if source_path_prefix and not dry_run:
|
||||
# Delete ImportTask rows whose source_path is under the bad mount
|
||||
# prefix. These are mostly orphaned now (result_image_id was set
|
||||
# NULL by CASCADE) but their presence still blocks the
|
||||
# idempotency check in scan_directory if the operator remounts
|
||||
# the same prefix.
|
||||
like_pattern = source_path_prefix.rstrip("/") + "/%"
|
||||
result = await db.execute(
|
||||
delete(ImportTask).where(ImportTask.source_path.like(like_pattern))
|
||||
)
|
||||
import_tasks_deleted = result.rowcount or 0
|
||||
await db.commit()
|
||||
|
||||
# Sweep ImportBatch rows that are now empty.
|
||||
empty_batches_deleted = 0
|
||||
if not dry_run:
|
||||
empty_batch_ids = (await db.execute(
|
||||
select(ImportBatch.id).where(
|
||||
~select(ImportTask.id)
|
||||
.where(ImportTask.batch_id == ImportBatch.id)
|
||||
.exists()
|
||||
)
|
||||
)).scalars().all()
|
||||
if empty_batch_ids:
|
||||
result = await db.execute(
|
||||
delete(ImportBatch).where(ImportBatch.id.in_(empty_batch_ids))
|
||||
)
|
||||
empty_batches_deleted = result.rowcount or 0
|
||||
await db.commit()
|
||||
|
||||
# Finally, the artist row.
|
||||
if not dry_run:
|
||||
await db.execute(delete(Artist).where(Artist.id == artist_id))
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"counts": counts,
|
||||
"artist": {"id": artist_id, "name": artist_name, "slug": slug},
|
||||
"summary": {
|
||||
"images_targeted": total_images,
|
||||
"images_deleted": images_deleted,
|
||||
"files_deleted": files_deleted,
|
||||
"thumbs_deleted": thumbs_deleted,
|
||||
"import_tasks_deleted": import_tasks_deleted,
|
||||
"empty_batches_deleted": empty_batches_deleted,
|
||||
"dry_run": dry_run,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Process-level sync engine for Celery task modules.
|
||||
|
||||
Each task module used to call ``create_engine(...)`` on every invocation,
|
||||
which leaked engines (and Postgres connections) — high-fire-rate tasks
|
||||
like ``import_media_file`` would exhaust ``max_connections`` within
|
||||
minutes during a bulk migration.
|
||||
|
||||
This module owns one engine per process. Celery prefork forks before any
|
||||
task runs, so each worker process lazily initializes its own engine on
|
||||
the first task and reuses it for the rest of its life.
|
||||
"""
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..config import get_config
|
||||
|
||||
_ENGINE = None
|
||||
_SESSIONMAKER = None
|
||||
|
||||
|
||||
def sync_session_factory():
|
||||
"""Return a process-wide ``sessionmaker`` bound to a single engine."""
|
||||
global _ENGINE, _SESSIONMAKER
|
||||
if _SESSIONMAKER is None:
|
||||
cfg = get_config()
|
||||
_ENGINE = create_engine(
|
||||
cfg.database_url_sync,
|
||||
future=True,
|
||||
pool_pre_ping=True,
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_recycle=300,
|
||||
)
|
||||
_SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False)
|
||||
return _SESSIONMAKER
|
||||
@@ -5,21 +5,13 @@ updates the ImportTask state machine + ImportBatch counters atomically.
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, select, update
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import ImportBatch, ImportSettings, ImportTask
|
||||
from ..services.importer import Importer
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||
return sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
@@ -6,13 +6,12 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import create_engine, delete, select, update
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import delete, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask
|
||||
from ..utils.phash import compute_phash
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,12 +22,6 @@ VERIFY_PAGE = 200
|
||||
FFPROBE_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||
return sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
|
||||
|
||||
@@ -4,7 +4,7 @@ Dispatches to the right migrator based on `kind`. Updates MigrationRun
|
||||
row's status/counts/finished_at as it runs. Failures set status='error'
|
||||
with the error message preserved.
|
||||
|
||||
kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback
|
||||
kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback, cleanup
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -21,6 +21,7 @@ from ..config import get_config
|
||||
from ..models import MigrationRun
|
||||
from ..services.credential_crypto import CredentialCrypto
|
||||
from ..services.migrators import backup as backup_mod
|
||||
from ..services.migrators import cleanup as cleanup_mod
|
||||
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
||||
from ..services.migrators import rollback as rollback_mod
|
||||
|
||||
@@ -145,6 +146,26 @@ async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
||||
)
|
||||
return {"checks": checks, "sample": sample}
|
||||
|
||||
elif kind == "cleanup":
|
||||
slug = params.get("slug")
|
||||
if not slug:
|
||||
raise ValueError("cleanup requires params.slug")
|
||||
result = await cleanup_mod.cleanup_artist_async(
|
||||
db, slug=slug, images_root=IMAGES_ROOT,
|
||||
dry_run=params.get("dry_run", False),
|
||||
source_path_prefix=params.get("source_path_prefix"),
|
||||
)
|
||||
await _update_run(
|
||||
db, run_id, status="ok",
|
||||
counts=result["counts"],
|
||||
finished_at=datetime.now(UTC),
|
||||
metadata_patch={
|
||||
"artist": result["artist"],
|
||||
"summary": result["summary"],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
elif kind == "rollback":
|
||||
result = rollback_mod.rollback_to_pre_migration(
|
||||
db_url=get_config().database_url_sync,
|
||||
|
||||
@@ -8,23 +8,16 @@ apply_allowlist_tags sweeps which are 'maintenance' lane. Sync sessions
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import ImageRecord, MLSettings
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"}
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||
return sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
def _is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
@@ -11,14 +11,14 @@ import asyncio
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
||||
from ..services.scheduler_service import select_due_sources
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
|
||||
def _iter_import_files(import_root: Path):
|
||||
@@ -35,12 +35,6 @@ def _iter_import_files(import_root: Path):
|
||||
yield entry
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||
return sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True)
|
||||
def scan_directory(self, triggered_by: str = "manual",
|
||||
mode: str = "quick") -> int:
|
||||
@@ -65,16 +59,34 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
session.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
# Skip-set: any source_path that already has a non-failed ImportTask
|
||||
# row. Re-running scan_directory must not re-enqueue files the
|
||||
# importer has already handled (or is currently handling); doing so
|
||||
# creates duplicate work and inflates the queue. Failed prior tasks
|
||||
# are eligible for retry.
|
||||
non_failed_existing = set(session.execute(
|
||||
select(ImportTask.source_path).where(
|
||||
ImportTask.status.in_(
|
||||
["pending", "queued", "processing", "complete", "skipped"]
|
||||
),
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
# Walk and enumerate.
|
||||
files_seen = 0
|
||||
files_skipped_existing = 0
|
||||
for entry in _iter_import_files(import_root):
|
||||
entry_str = str(entry)
|
||||
if entry_str in non_failed_existing:
|
||||
files_skipped_existing += 1
|
||||
continue
|
||||
try:
|
||||
size = entry.stat().st_size
|
||||
except OSError:
|
||||
size = None
|
||||
task = ImportTask(
|
||||
batch_id=batch_id,
|
||||
source_path=str(entry),
|
||||
source_path=entry_str,
|
||||
task_type="media",
|
||||
status="pending",
|
||||
size_bytes=size,
|
||||
@@ -83,6 +95,15 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
files_seen += 1
|
||||
|
||||
batch.total_files = files_seen
|
||||
# If the walk enqueued nothing (every file was already on a
|
||||
# non-failed ImportTask from a prior scan), there's no
|
||||
# import_media_file message that would ever flip this batch to
|
||||
# 'complete' — finalize it now so the active-batch query in
|
||||
# /api/system/stats doesn't get stuck reporting all-zero
|
||||
# counters from an empty scan.
|
||||
if files_seen == 0:
|
||||
batch.status = "complete"
|
||||
batch.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
|
||||
# Now enqueue import_media_file for each pending task.
|
||||
|
||||
@@ -7,24 +7,15 @@ so they deserve their own queue lane.
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import ImageRecord
|
||||
from ..services.importer import is_video
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||
return sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.thumbnail.generate_thumbnail", bind=True)
|
||||
def generate_thumbnail(self, image_id: int) -> dict:
|
||||
SessionLocal = _sync_session_factory()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
<AppShell>
|
||||
<RouterView />
|
||||
</AppShell>
|
||||
<ImageViewer v-if="modal.isOpen" @close="modal.close()" />
|
||||
<AppSnackbar ref="snackbar" />
|
||||
</v-app>
|
||||
</template>
|
||||
@@ -11,7 +12,10 @@
|
||||
import { onMounted, ref } from 'vue'
|
||||
import AppShell from './components/AppShell.vue'
|
||||
import AppSnackbar from './components/AppSnackbar.vue'
|
||||
import ImageViewer from './components/modal/ImageViewer.vue'
|
||||
import { useModalStore } from './stores/modal.js'
|
||||
|
||||
const modal = useModalStore()
|
||||
const snackbar = ref(null)
|
||||
|
||||
onMounted(() => {
|
||||
|
||||
@@ -55,7 +55,13 @@ const health = computed(() => {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1000;
|
||||
display: flex;
|
||||
/* Mirrored side columns (1fr / auto / 1fr) keep the link block dead-
|
||||
centered no matter what the teleport-slot on the right is rendering
|
||||
for the active view (Gallery: Select, Showcase: Shuffle, others: ∅).
|
||||
With a plain flex layout the links shifted left as soon as actions
|
||||
appeared. */
|
||||
display: grid;
|
||||
grid-template-columns: 1fr auto 1fr;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 0.75rem 1rem;
|
||||
@@ -87,7 +93,6 @@ const health = computed(() => {
|
||||
}
|
||||
|
||||
.fc-links {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
@@ -118,6 +123,7 @@ const health = computed(() => {
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
justify-self: start;
|
||||
}
|
||||
.fc-health {
|
||||
display: flex;
|
||||
@@ -129,5 +135,6 @@ const health = computed(() => {
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
flex-shrink: 0;
|
||||
justify-self: end;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -86,7 +86,7 @@ function imagesForGroup(group) {
|
||||
}
|
||||
.fc-gallery-grid__items {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: 8px;
|
||||
}
|
||||
.fc-gallery-grid__sentinel {
|
||||
@@ -101,7 +101,7 @@ function imagesForGroup(group) {
|
||||
}
|
||||
.fc-gallery-grid__skeleton {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: 8px;
|
||||
}
|
||||
.fc-gallery-grid__skeleton-item {
|
||||
@@ -124,7 +124,7 @@ function imagesForGroup(group) {
|
||||
@media (max-width: 600px) {
|
||||
.fc-gallery-grid__items,
|
||||
.fc-gallery-grid__skeleton {
|
||||
grid-template-columns: repeat(auto-fill, minmax(120px, 1fr));
|
||||
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
|
||||
gap: 4px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
<template>
|
||||
<div
|
||||
ref="canvasEl"
|
||||
class="fc-canvas"
|
||||
:class="{ 'fc-canvas--zoomed': panZoom.state.scale > 1 }"
|
||||
@wheel="panZoom.handlers.onWheel"
|
||||
@pointerdown="panZoom.handlers.onPointerDown"
|
||||
@pointermove="panZoom.handlers.onPointerMove"
|
||||
@pointerup="panZoom.handlers.onPointerUp"
|
||||
@click="panZoom.handlers.onClick"
|
||||
@click="onCanvasClick"
|
||||
>
|
||||
<img
|
||||
ref="imgEl"
|
||||
:src="src" :alt="alt"
|
||||
:style="{
|
||||
transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})`
|
||||
@@ -19,21 +21,51 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { watch } from 'vue'
|
||||
import { ref, watch } from 'vue'
|
||||
import { usePanZoom } from '../../composables/usePanZoom.js'
|
||||
|
||||
const props = defineProps({ src: String, alt: String })
|
||||
const emit = defineEmits(['close-request'])
|
||||
const panZoom = usePanZoom()
|
||||
|
||||
const canvasEl = ref(null)
|
||||
const imgEl = ref(null)
|
||||
|
||||
// Reset zoom when the src changes (prev/next nav).
|
||||
watch(() => props.src, () => panZoom.reset())
|
||||
|
||||
// Click on the image → toggle zoom (IR/expected behavior). Click on
|
||||
// the haze area around the image → request close. We use the image's
|
||||
// own bounding rect rather than @click.self because the <img> sets
|
||||
// pointer-events: none for the pan-drag pathway, so the click always
|
||||
// targets the canvas div regardless of where the cursor was.
|
||||
function onCanvasClick(ev) {
|
||||
if (panZoom.state.scale > 1) {
|
||||
// When zoomed, any click resets zoom — preserve old behavior.
|
||||
panZoom.handlers.onClick(ev)
|
||||
return
|
||||
}
|
||||
const img = imgEl.value
|
||||
if (!img) return
|
||||
const r = img.getBoundingClientRect()
|
||||
const inside =
|
||||
ev.clientX >= r.left && ev.clientX <= r.right &&
|
||||
ev.clientY >= r.top && ev.clientY <= r.bottom
|
||||
if (inside) {
|
||||
panZoom.handlers.onClick(ev)
|
||||
} else {
|
||||
emit('close-request')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-canvas {
|
||||
flex: 1;
|
||||
position: relative;
|
||||
background: rgb(var(--v-theme-background));
|
||||
/* Transparent so the viewer's haze shows through to the image area
|
||||
instead of a solid panel sitting on top of it. */
|
||||
background: transparent;
|
||||
overflow: hidden;
|
||||
cursor: zoom-in;
|
||||
user-select: none;
|
||||
|
||||
@@ -39,9 +39,11 @@
|
||||
<template v-else-if="modal.current">
|
||||
<ImageCanvas
|
||||
v-if="!isVideo" :src="modal.current.image_url" :alt="`Image ${modal.current.id}`"
|
||||
@close-request="$emit('close')"
|
||||
/>
|
||||
<VideoCanvas
|
||||
v-else :src="modal.current.image_url" :mime="modal.current.mime"
|
||||
@close-request="$emit('close')"
|
||||
/>
|
||||
</template>
|
||||
<v-alert v-else-if="modal.error" type="error" variant="tonal">
|
||||
@@ -123,7 +125,11 @@ function isTextEntry(el) {
|
||||
<style scoped>
|
||||
.fc-viewer {
|
||||
position: fixed; inset: 0; z-index: 2000;
|
||||
background: rgba(20, 23, 26, 0.96);
|
||||
/* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav,
|
||||
mid-opacity + blur so the page behind shows through faintly. */
|
||||
background: rgba(20, 23, 26, 0.65);
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
outline: none;
|
||||
display: flex; flex-direction: column;
|
||||
}
|
||||
@@ -152,8 +158,12 @@ function isTextEntry(el) {
|
||||
flex: 1; display: flex; min-height: 0;
|
||||
}
|
||||
.fc-viewer__media {
|
||||
flex: 1; display: flex; align-items: center; justify-content: center;
|
||||
min-width: 0;
|
||||
/* Let the canvas fill us; the canvas does its own centering. Both
|
||||
min-* are needed so this child can shrink inside its flex parents
|
||||
(without them, max-height/max-width on the <img> have nothing to
|
||||
bound against and the image overflows the viewport). */
|
||||
flex: 1; display: flex;
|
||||
min-width: 0; min-height: 0;
|
||||
}
|
||||
.fc-viewer__side {
|
||||
width: 320px; flex-shrink: 0;
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
<template>
|
||||
<div class="fc-canvas fc-canvas--video">
|
||||
<div class="fc-canvas fc-canvas--video" @click.self="$emit('close-request')">
|
||||
<video
|
||||
v-if="playable" :src="src" controls playsinline preload="metadata"
|
||||
class="fc-canvas__video"
|
||||
@click.stop
|
||||
/>
|
||||
<div v-else class="fc-canvas__unsupported">
|
||||
<div v-else class="fc-canvas__unsupported" @click.stop>
|
||||
<v-icon size="56" icon="mdi-video-off-outline" />
|
||||
<h3 class="fc-canvas__unsupported-title">Format not browser-playable</h3>
|
||||
<p>
|
||||
@@ -23,6 +24,7 @@
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
|
||||
defineEmits(['close-request'])
|
||||
const props = defineProps({ src: String, mime: String })
|
||||
|
||||
const BROWSER_PLAYABLE = new Set([
|
||||
@@ -34,7 +36,9 @@ const playable = computed(() => BROWSER_PLAYABLE.has(props.mime))
|
||||
<style scoped>
|
||||
.fc-canvas {
|
||||
flex: 1;
|
||||
background: rgb(var(--v-theme-background));
|
||||
/* Transparent so the viewer's haze shows through (parallels
|
||||
ImageCanvas — both sit inside ImageViewer's blurred backdrop). */
|
||||
background: transparent;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
}
|
||||
.fc-canvas__video { max-width: 100%; max-height: 100%; }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<v-row dense>
|
||||
<v-col v-for="card in cards" :key="card.label" cols="12" sm="6" md="4" lg="2">
|
||||
<v-col v-for="card in cards" :key="card.label" cols="12" sm="6" md="4" lg="3" xl="2">
|
||||
<v-card class="fc-stat">
|
||||
<v-card-text>
|
||||
<div class="fc-stat__label text-caption">{{ card.label }}</div>
|
||||
@@ -33,15 +33,17 @@ const cards = computed(() => {
|
||||
{ label: 'Total images', value: '—' },
|
||||
{ label: 'Total tags', value: '—' },
|
||||
{ label: 'Storage used', value: '—' },
|
||||
{ label: 'Subscriptions', value: '—' },
|
||||
{ label: 'Pending tasks', value: '—' },
|
||||
{ label: 'Failed tasks', value: '—' }
|
||||
]
|
||||
return [
|
||||
{ label: 'Total images', value: s.total_images.toLocaleString() },
|
||||
{ label: 'Total tags', value: s.total_tags.toLocaleString() },
|
||||
{ label: 'Storage used', value: formatBytes(s.storage_bytes) },
|
||||
{ label: 'Pending', value: (s.tasks.pending + s.tasks.queued).toLocaleString() },
|
||||
{ label: 'Failed', value: s.tasks.failed.toLocaleString() }
|
||||
{ label: 'Total images', value: s.total_images.toLocaleString() },
|
||||
{ label: 'Total tags', value: s.total_tags.toLocaleString() },
|
||||
{ label: 'Storage used', value: formatBytes(s.storage_bytes) },
|
||||
{ label: 'Subscriptions', value: (s.subscription_count ?? 0).toLocaleString() },
|
||||
{ label: 'Pending', value: (s.tasks.pending + s.tasks.queued).toLocaleString() },
|
||||
{ label: 'Failed', value: s.tasks.failed.toLocaleString() }
|
||||
]
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
<template>
|
||||
<section class="fc-artist-section">
|
||||
<header class="fc-artist-section__head" @click="$emit('toggle')">
|
||||
<v-icon :icon="open ? 'mdi-chevron-down' : 'mdi-chevron-right'" size="small" />
|
||||
<span class="fc-artist-section__name">{{ artist.name }}</span>
|
||||
<span class="fc-artist-section__meta">
|
||||
{{ sources.length }} source{{ sources.length === 1 ? '' : 's' }}
|
||||
<template v-if="latestChecked">
|
||||
· last check {{ latestChecked }}
|
||||
</template>
|
||||
</span>
|
||||
</header>
|
||||
<div v-if="open" class="fc-artist-section__body">
|
||||
<SourceRow
|
||||
v-for="s in sources" :key="s.id" :source="s"
|
||||
:checking="checkingIds.has(s.id)"
|
||||
@edit="$emit('edit', $event)"
|
||||
@remove="$emit('remove', $event)"
|
||||
@toggle="$emit('toggle-source', $event)"
|
||||
@check="$emit('check', $event)"
|
||||
/>
|
||||
<v-btn
|
||||
size="small" variant="text" prepend-icon="mdi-plus"
|
||||
class="fc-artist-section__add"
|
||||
@click="$emit('add-source', artist)"
|
||||
>Add source to {{ artist.name }}</v-btn>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import SourceRow from './SourceRow.vue'
|
||||
|
||||
const props = defineProps({
|
||||
artist: { type: Object, required: true },
|
||||
sources: { type: Array, required: true },
|
||||
open: { type: Boolean, default: false },
|
||||
checkingIds: { type: Set, default: () => new Set() },
|
||||
})
|
||||
defineEmits(['toggle', 'edit', 'remove', 'toggle-source', 'add-source', 'check'])
|
||||
|
||||
const latestChecked = computed(() => {
|
||||
const dates = props.sources.map(s => s.last_checked_at).filter(Boolean)
|
||||
if (dates.length === 0) return null
|
||||
const latest = dates.sort().slice(-1)[0]
|
||||
return latest.slice(0, 10)
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-artist-section { border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.18); }
|
||||
.fc-artist-section__head {
|
||||
display: flex; align-items: center; gap: 0.5rem;
|
||||
padding: 0.75rem 0; cursor: pointer; user-select: none;
|
||||
}
|
||||
.fc-artist-section__name { font-weight: 600; }
|
||||
.fc-artist-section__meta {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.fc-artist-section__body { padding: 0 0 0.75rem 1.5rem; }
|
||||
.fc-artist-section__add { margin-top: 0.25rem; }
|
||||
</style>
|
||||
@@ -1,25 +1,63 @@
|
||||
<template>
|
||||
<div class="fc-source-row">
|
||||
<SourceHealthDot :source="source" :warning-threshold="warningThreshold" />
|
||||
<v-chip size="x-small" variant="tonal" class="fc-source-row__platform">
|
||||
{{ source.platform }}
|
||||
</v-chip>
|
||||
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url">
|
||||
{{ source.url }}
|
||||
</a>
|
||||
<v-switch
|
||||
:model-value="source.enabled"
|
||||
density="compact" hide-details color="accent"
|
||||
@update:model-value="onToggleEnabled"
|
||||
/>
|
||||
<v-btn
|
||||
icon="mdi-play" size="x-small" variant="text"
|
||||
:loading="checking"
|
||||
@click="$emit('check', source)"
|
||||
/>
|
||||
<v-btn icon="mdi-pencil" size="x-small" variant="text" @click="$emit('edit', source)" />
|
||||
<v-btn icon="mdi-close" size="x-small" variant="text" @click="$emit('remove', source)" />
|
||||
</div>
|
||||
<tr class="fc-source-row">
|
||||
<td class="fc-source-row__health">
|
||||
<SourceHealthDot :source="source" :warning-threshold="warningThreshold" />
|
||||
</td>
|
||||
<td>
|
||||
<v-chip size="x-small" variant="tonal" label>{{ source.platform }}</v-chip>
|
||||
</td>
|
||||
<td class="fc-source-row__url-cell">
|
||||
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
|
||||
@click.stop>
|
||||
{{ source.url }}
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<v-switch
|
||||
:model-value="source.enabled"
|
||||
density="compact" hide-details color="accent"
|
||||
@click.stop
|
||||
@update:model-value="onToggleEnabled"
|
||||
/>
|
||||
</td>
|
||||
<td class="fc-source-row__when">
|
||||
{{ formatRelative(source.last_checked_at) }}
|
||||
</td>
|
||||
<td class="fc-source-row__when">
|
||||
{{ formatRelative(source.next_check_at, { future: true }) }}
|
||||
</td>
|
||||
<td>
|
||||
<v-chip
|
||||
v-if="(source.consecutive_failures || 0) > 0"
|
||||
size="x-small" color="error" variant="tonal" label
|
||||
>{{ source.consecutive_failures }}</v-chip>
|
||||
<span v-else class="fc-source-row__zero">0</span>
|
||||
</td>
|
||||
<td class="fc-source-row__actions">
|
||||
<v-btn
|
||||
icon="mdi-play" size="x-small" variant="text"
|
||||
:loading="checking"
|
||||
@click.stop="$emit('check', source)"
|
||||
>
|
||||
<v-icon>mdi-play</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Check now</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon="mdi-pencil" size="x-small" variant="text"
|
||||
@click.stop="$emit('edit', source)"
|
||||
>
|
||||
<v-icon>mdi-pencil</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Edit</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon="mdi-close" size="x-small" variant="text" color="error"
|
||||
@click.stop="$emit('remove', source)"
|
||||
>
|
||||
<v-icon>mdi-close</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Remove</v-tooltip>
|
||||
</v-btn>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
@@ -31,25 +69,59 @@ const props = defineProps({
|
||||
warningThreshold: { type: Number, default: 5 },
|
||||
})
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check'])
|
||||
|
||||
function onToggleEnabled(value) {
|
||||
emit('toggle', { source: props.source, enabled: value })
|
||||
}
|
||||
|
||||
function formatRelative(iso, opts = {}) {
|
||||
if (!iso) return opts.future ? '—' : 'Never'
|
||||
const then = new Date(iso).getTime()
|
||||
const now = Date.now()
|
||||
const diff = (then - now) / 1000 // seconds; positive = future, negative = past
|
||||
const abs = Math.abs(diff)
|
||||
|
||||
let body
|
||||
if (abs < 60) body = `${Math.floor(abs)}s`
|
||||
else if (abs < 3600) body = `${Math.floor(abs / 60)}m`
|
||||
else if (abs < 86400) body = `${Math.floor(abs / 3600)}h`
|
||||
else body = `${Math.floor(abs / 86400)}d`
|
||||
|
||||
if (opts.future) return diff <= 0 ? 'imminent' : `in ${body}`
|
||||
return `${body} ago`
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-source-row {
|
||||
display: grid;
|
||||
grid-template-columns: auto 96px 1fr auto auto auto auto;
|
||||
gap: 0.75rem;
|
||||
align-items: center;
|
||||
padding: 0.4rem 0;
|
||||
.fc-source-row__health {
|
||||
width: 24px;
|
||||
padding-right: 0 !important;
|
||||
}
|
||||
.fc-source-row__url-cell {
|
||||
max-width: 400px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fc-source-row__url {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.fc-source-row__url:hover { color: rgb(var(--v-theme-accent)); }
|
||||
.fc-source-row__when {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
white-space: nowrap;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.fc-source-row__zero {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
opacity: 0.6;
|
||||
}
|
||||
.fc-source-row__actions {
|
||||
white-space: nowrap;
|
||||
text-align: right;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -29,10 +29,11 @@ export function distributeIntoColumns(items, columnCount) {
|
||||
}
|
||||
|
||||
// Reactive column count from container width. Breakpoints mirror the
|
||||
// gallery grid intent: ~260px target column width, min 1 column.
|
||||
// gallery grid intent: ~390px target column width, min 1 column. Bumped
|
||||
// from 260px on 2026-05-23 per dogfood UX feedback (thumbs were too small).
|
||||
export function columnCountForWidth(width) {
|
||||
if (!width || width < 0) return 1
|
||||
return Math.max(1, Math.floor(width / 260))
|
||||
return Math.max(1, Math.floor(width / 390))
|
||||
}
|
||||
|
||||
export function usePolyMasonry(containerRef) {
|
||||
|
||||
@@ -104,11 +104,13 @@
|
||||
import { computed, watch } from 'vue'
|
||||
import { useRoute, useRouter, RouterLink } from 'vue-router'
|
||||
import { useArtistStore } from '../stores/artist.js'
|
||||
import { useModalStore } from '../stores/modal.js'
|
||||
import MasonryGrid from '../components/discovery/MasonryGrid.vue'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const store = useArtistStore()
|
||||
const modal = useModalStore()
|
||||
|
||||
const slug = computed(() => route.params.slug)
|
||||
|
||||
@@ -136,7 +138,7 @@ const sparkPoints = computed(() => {
|
||||
})
|
||||
|
||||
function openImage(id) {
|
||||
router.push({ name: 'gallery', query: { image: id } })
|
||||
modal.open(id)
|
||||
}
|
||||
function openTag(tagId) {
|
||||
router.push({ name: 'gallery', query: { tag_id: tagId } })
|
||||
|
||||
@@ -18,10 +18,6 @@
|
||||
<TimelineSidebar v-if="store.images.length > 0" class="fc-gallery-layout__sidebar" />
|
||||
</div>
|
||||
|
||||
<ImageViewer
|
||||
v-if="modal.currentImageId !== null"
|
||||
@close="closeImage"
|
||||
/>
|
||||
<BulkEditorPanel />
|
||||
</v-container>
|
||||
</template>
|
||||
@@ -35,7 +31,6 @@ import GalleryGrid from '../components/gallery/GalleryGrid.vue'
|
||||
import TimelineSidebar from '../components/gallery/TimelineSidebar.vue'
|
||||
import EmptyState from '../components/gallery/EmptyState.vue'
|
||||
import PostInfoHeader from '../components/gallery/PostInfoHeader.vue'
|
||||
import ImageViewer from '../components/modal/ImageViewer.vue'
|
||||
import BulkEditorPanel from '../components/gallery/BulkEditorPanel.vue'
|
||||
import { useGallerySelectionStore } from '../stores/gallerySelection.js'
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<v-container class="py-6">
|
||||
<v-container fluid class="py-6">
|
||||
<v-tabs v-model="tab" color="accent" class="mb-4">
|
||||
<v-tab value="overview">Overview</v-tab>
|
||||
<v-tab value="import">Import</v-tab>
|
||||
@@ -57,7 +57,12 @@ function startPolling() {
|
||||
pollId = setInterval(() => {
|
||||
if (!document.hidden) {
|
||||
system.refreshStats()
|
||||
if (tab.value === 'import') importStore.refreshStatus()
|
||||
if (tab.value === 'import') {
|
||||
importStore.refreshStatus()
|
||||
// Refresh the task list while a batch is in flight so the UI
|
||||
// doesn't sit stale next to a ticking imported/skipped counter.
|
||||
if (importStore.activeBatch) importStore.loadTasks(true)
|
||||
}
|
||||
}
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
@@ -26,16 +26,16 @@
|
||||
|
||||
<script setup>
|
||||
import { onMounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { useShowcaseStore } from '../stores/showcase.js'
|
||||
import { useModalStore } from '../stores/modal.js'
|
||||
import MasonryGrid from '../components/discovery/MasonryGrid.vue'
|
||||
|
||||
const store = useShowcaseStore()
|
||||
const router = useRouter()
|
||||
const modal = useModalStore()
|
||||
|
||||
onMounted(() => { if (store.images.length === 0) store.fetchPage() })
|
||||
|
||||
function openImage(id) {
|
||||
router.push({ name: 'gallery', query: { image: id } })
|
||||
modal.open(id)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -8,9 +8,13 @@
|
||||
New artist
|
||||
</v-btn>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" size="small" @click="expandAll = !expandAll">
|
||||
{{ expandAll ? 'Collapse all' : 'Expand all' }}
|
||||
</v-btn>
|
||||
<v-text-field
|
||||
v-model="search"
|
||||
density="compact" variant="outlined" hide-details clearable
|
||||
prepend-inner-icon="mdi-magnify"
|
||||
placeholder="Search subscriptions"
|
||||
style="max-width: 320px"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mt-4">
|
||||
@@ -21,24 +25,121 @@
|
||||
<v-progress-circular indeterminate color="accent" size="36" />
|
||||
</div>
|
||||
|
||||
<div v-else-if="groups.length === 0" class="fc-subs__empty">
|
||||
<p>No subscriptions yet. Add your first artist.</p>
|
||||
<div v-else-if="filteredGroups.length === 0" class="fc-subs__empty">
|
||||
<p v-if="groups.length === 0">No subscriptions yet. Add your first artist.</p>
|
||||
<p v-else>No subscriptions match "{{ search }}".</p>
|
||||
</div>
|
||||
|
||||
<div v-else>
|
||||
<ArtistSection
|
||||
v-for="g in groups" :key="g.artist.id"
|
||||
:artist="g.artist" :sources="g.sources"
|
||||
:open="isOpen(g.artist.id)"
|
||||
:checking-ids="store.checkingIds"
|
||||
@toggle="toggleSection(g.artist.id)"
|
||||
@edit="openEditSource"
|
||||
@remove="removeSource"
|
||||
@toggle-source="toggleSourceEnabled"
|
||||
@add-source="openAddSource"
|
||||
@check="onCheck"
|
||||
/>
|
||||
</div>
|
||||
<v-card v-else class="fc-subs__card" variant="outlined">
|
||||
<v-data-table
|
||||
:headers="headers"
|
||||
:items="filteredGroups"
|
||||
item-value="key"
|
||||
v-model:expanded="expanded"
|
||||
:items-per-page="50"
|
||||
:items-per-page-options="ITEMS_PER_PAGE_OPTIONS"
|
||||
density="comfortable"
|
||||
hover
|
||||
show-expand
|
||||
@click:row="onRowClick"
|
||||
>
|
||||
<template #item.name="{ item }">
|
||||
<span class="fc-subs__name">{{ item.artist.name }}</span>
|
||||
</template>
|
||||
|
||||
<template #item.sources_count="{ item }">
|
||||
<v-chip size="x-small" variant="tonal" label>
|
||||
{{ item.sources.length }} source{{ item.sources.length === 1 ? '' : 's' }}
|
||||
</v-chip>
|
||||
</template>
|
||||
|
||||
<template #item.health="{ item }">
|
||||
<SourceHealthDot
|
||||
v-if="item.worstSource"
|
||||
:source="item.worstSource"
|
||||
:warning-threshold="failureThreshold"
|
||||
/>
|
||||
<span v-else class="fc-subs__zero">—</span>
|
||||
</template>
|
||||
|
||||
<template #item.last_activity="{ item }">
|
||||
<span class="fc-subs__when">
|
||||
{{ formatRelative(item.lastActivity) }}
|
||||
</span>
|
||||
</template>
|
||||
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:loading="anyChecking(item.sources)"
|
||||
@click.stop="checkAll(item)"
|
||||
>
|
||||
<v-icon>mdi-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Check all sources</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
@click.stop="openAddSource(item.artist)"
|
||||
>
|
||||
<v-icon>mdi-plus</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Add source</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:to="`/posts?artist_id=${item.artist.id}`"
|
||||
@click.stop
|
||||
>
|
||||
<v-icon>mdi-rss</v-icon>
|
||||
<v-tooltip activator="parent" location="top">View posts</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon size="small" variant="text"
|
||||
:to="`/artist/${item.artist.slug}`"
|
||||
@click.stop
|
||||
>
|
||||
<v-icon>mdi-account</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Open artist page</v-tooltip>
|
||||
</v-btn>
|
||||
</template>
|
||||
|
||||
<template #expanded-row="{ columns, item }">
|
||||
<tr class="fc-subs__sources-row">
|
||||
<td :colspan="columns.length" class="fc-subs__sources-cell">
|
||||
<v-table density="compact" class="fc-subs__sources-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Platform</th>
|
||||
<th>URL</th>
|
||||
<th>Enabled</th>
|
||||
<th>Last check</th>
|
||||
<th>Next check</th>
|
||||
<th>Errors</th>
|
||||
<th class="text-right">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<SourceRow
|
||||
v-for="s in item.sources" :key="s.id" :source="s"
|
||||
:checking="store.checkingIds.has(s.id)"
|
||||
:warning-threshold="failureThreshold"
|
||||
@edit="openEditSource"
|
||||
@remove="removeSource"
|
||||
@toggle="toggleSourceEnabled"
|
||||
@check="onCheck"
|
||||
/>
|
||||
<tr v-if="item.sources.length === 0">
|
||||
<td colspan="8" class="fc-subs__sources-empty">
|
||||
No sources yet. Click + to add one.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
</v-data-table>
|
||||
</v-card>
|
||||
|
||||
<SourceFormDialog
|
||||
v-model="showSourceDialog"
|
||||
@@ -55,52 +156,140 @@ import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { useSourcesStore } from '../stores/sources.js'
|
||||
import { usePlatformsStore } from '../stores/platforms.js'
|
||||
import ArtistSection from '../components/subscriptions/ArtistSection.vue'
|
||||
import { useImportStore } from '../stores/import.js'
|
||||
import SourceRow from '../components/subscriptions/SourceRow.vue'
|
||||
import SourceHealthDot from '../components/subscriptions/SourceHealthDot.vue'
|
||||
import SourceFormDialog from '../components/subscriptions/SourceFormDialog.vue'
|
||||
import ArtistCreateDialog from '../components/subscriptions/ArtistCreateDialog.vue'
|
||||
|
||||
const ITEMS_PER_PAGE_OPTIONS = [
|
||||
{ value: 25, title: '25' },
|
||||
{ value: 50, title: '50' },
|
||||
{ value: 100, title: '100' },
|
||||
{ value: -1, title: 'All' },
|
||||
]
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const store = useSourcesStore()
|
||||
const platformsStore = usePlatformsStore()
|
||||
const importStore = useImportStore()
|
||||
|
||||
const search = ref('')
|
||||
const expanded = ref([])
|
||||
const showSourceDialog = ref(false)
|
||||
const editingSource = ref(null)
|
||||
const editingArtist = ref(null)
|
||||
const showArtistDialog = ref(false)
|
||||
|
||||
const artistFilter = computed(() => {
|
||||
const raw = route.query.artist_id
|
||||
return raw == null ? null : Number(raw)
|
||||
})
|
||||
|
||||
const expandAll = ref(false)
|
||||
const openSections = ref(new Set())
|
||||
const showSourceDialog = ref(false)
|
||||
const editingSource = ref(null)
|
||||
const editingArtist = ref(null)
|
||||
const showArtistDialog = ref(false)
|
||||
const failureThreshold = computed(() =>
|
||||
importStore.settings?.download_failure_warning_threshold ?? 5
|
||||
)
|
||||
|
||||
async function refresh() {
|
||||
await store.loadAll()
|
||||
await platformsStore.loadAll()
|
||||
if (artistFilter.value != null) {
|
||||
openSections.value = new Set([artistFilter.value])
|
||||
expandAll.value = false
|
||||
}
|
||||
if (!importStore.settings) await importStore.loadSettings()
|
||||
}
|
||||
|
||||
onMounted(refresh)
|
||||
onMounted(() => {
|
||||
refresh()
|
||||
if (artistFilter.value != null) {
|
||||
// Pre-expand the row that the deep-link refers to.
|
||||
expanded.value = [`artist-${artistFilter.value}`]
|
||||
}
|
||||
})
|
||||
watch(() => route.query.artist_id, refresh)
|
||||
|
||||
const headers = [
|
||||
{ title: 'Subscription', key: 'name', sortable: true, align: 'start' },
|
||||
{ title: 'Sources', key: 'sources_count', sortable: true, align: 'start', width: 110 },
|
||||
{ title: 'Health', key: 'health', sortable: false, align: 'start', width: 80 },
|
||||
{ title: 'Last activity',key: 'last_activity', sortable: true, align: 'start', width: 140 },
|
||||
{ title: 'Actions', key: 'actions', sortable: false, align: 'end', width: 200 },
|
||||
]
|
||||
|
||||
const groups = computed(() => {
|
||||
const all = store.sourcesByArtistGrouped()
|
||||
if (artistFilter.value == null) return all
|
||||
return all.filter(g => g.artist.id === artistFilter.value)
|
||||
return all.map(g => {
|
||||
const worstSource = pickWorstSource(g.sources, failureThreshold.value)
|
||||
const lastActivity = pickLastActivity(g.sources)
|
||||
return {
|
||||
key: `artist-${g.artist.id}`,
|
||||
artist: g.artist,
|
||||
sources: g.sources,
|
||||
sources_count: g.sources.length,
|
||||
worstSource,
|
||||
lastActivity,
|
||||
name: g.artist.name, // for sortable column
|
||||
last_activity: lastActivity ?? '', // for sortable column
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
function isOpen(artistId) {
|
||||
return expandAll.value || openSections.value.has(artistId)
|
||||
const filteredGroups = computed(() => {
|
||||
let arr = groups.value
|
||||
if (artistFilter.value != null) {
|
||||
arr = arr.filter(g => g.artist.id === artistFilter.value)
|
||||
}
|
||||
const q = search.value?.trim().toLowerCase()
|
||||
if (q) {
|
||||
arr = arr.filter(g =>
|
||||
g.artist.name.toLowerCase().includes(q)
|
||||
|| g.sources.some(s => (s.url || '').toLowerCase().includes(q)
|
||||
|| (s.platform || '').toLowerCase().includes(q))
|
||||
)
|
||||
}
|
||||
return arr
|
||||
})
|
||||
|
||||
function pickLastActivity(sources) {
|
||||
let max = null
|
||||
for (const s of sources) {
|
||||
if (s.last_checked_at && (!max || s.last_checked_at > max)) max = s.last_checked_at
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
function toggleSection(artistId) {
|
||||
if (openSections.value.has(artistId)) openSections.value.delete(artistId)
|
||||
else openSections.value.add(artistId)
|
||||
function pickWorstSource(sources, threshold) {
|
||||
// Health order (worst → best): critical, warning, healthy, unchecked.
|
||||
// Picks the source with the worst level so the row's dot reflects the
|
||||
// worst-case state. Within a level, the first is fine.
|
||||
if (!sources || sources.length === 0) return null
|
||||
function level(s) {
|
||||
if (!s.last_checked_at) return 0 // unchecked
|
||||
const f = s.consecutive_failures || 0
|
||||
if (f === 0) return 1 // healthy
|
||||
if (f < threshold) return 2 // warning
|
||||
return 3 // critical
|
||||
}
|
||||
return sources.reduce((worst, s) =>
|
||||
level(s) > level(worst) ? s : worst,
|
||||
sources[0],
|
||||
)
|
||||
}
|
||||
|
||||
function formatRelative(iso) {
|
||||
if (!iso) return 'Never'
|
||||
const then = new Date(iso).getTime()
|
||||
const diff = (Date.now() - then) / 1000
|
||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||
return `${Math.floor(diff / 86400)}d ago`
|
||||
}
|
||||
|
||||
function onRowClick(_evt, { item, internalItem }) {
|
||||
// Toggle expansion on row click (in addition to the chevron).
|
||||
const key = item.key
|
||||
const idx = expanded.value.indexOf(key)
|
||||
if (idx === -1) expanded.value = [...expanded.value, key]
|
||||
else expanded.value = expanded.value.filter(k => k !== key)
|
||||
}
|
||||
|
||||
function openAddSource(artist) {
|
||||
@@ -132,7 +321,6 @@ async function onSourceSaved() {
|
||||
|
||||
function onArtistCreated(artist) {
|
||||
showArtistDialog.value = false
|
||||
// Move into Add Source for the new artist immediately.
|
||||
openAddSource(artist)
|
||||
}
|
||||
|
||||
@@ -158,6 +346,31 @@ async function onCheck(source) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAll(group) {
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
for (const s of group.sources) {
|
||||
if (!s.enabled) continue
|
||||
try {
|
||||
await store.checkNow(s.id)
|
||||
ok += 1
|
||||
} catch (e) {
|
||||
if (e?.body?.download_event_id) conflict += 1
|
||||
}
|
||||
}
|
||||
const parts = []
|
||||
if (ok) parts.push(`${ok} queued`)
|
||||
if (conflict) parts.push(`${conflict} already running`)
|
||||
globalThis.window?.__fcToast?.({
|
||||
text: parts.join(', ') || 'Nothing to check (no enabled sources)',
|
||||
type: 'info',
|
||||
})
|
||||
}
|
||||
|
||||
function anyChecking(sources) {
|
||||
return sources.some(s => store.checkingIds.has(s.id))
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@@ -169,4 +382,35 @@ async function onCheck(source) {
|
||||
display: flex; justify-content: center; padding: 2rem;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-subs__card {
|
||||
background: rgb(var(--v-theme-surface));
|
||||
}
|
||||
.fc-subs__name {
|
||||
font-weight: 600;
|
||||
}
|
||||
.fc-subs__when {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
white-space: nowrap;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.fc-subs__zero {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
opacity: 0.6;
|
||||
}
|
||||
.fc-subs__sources-row td {
|
||||
padding: 0 !important;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-subs__sources-cell {
|
||||
padding-left: 2rem !important;
|
||||
border-top: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||
}
|
||||
.fc-subs__sources-table {
|
||||
background: transparent !important;
|
||||
}
|
||||
.fc-subs__sources-empty {
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
text-align: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -76,7 +76,7 @@ async def test_system_stats_shape(client):
|
||||
resp = await client.get("/api/system/stats")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
for key in ("total_images", "total_tags", "storage_bytes", "tasks", "active_batch"):
|
||||
for key in ("total_images", "total_tags", "storage_bytes", "subscription_count", "tasks", "active_batch"):
|
||||
assert key in body
|
||||
for status in ("pending", "queued", "processing", "complete", "skipped", "failed"):
|
||||
assert status in body["tasks"]
|
||||
|
||||
Reference in New Issue
Block a user