"""Files in the artists' folders that the library has no record of. A download writes its files into the artist's folder and imports them in a later step. A run that stops between the two (a restart, a time limit) leaves complete files on disk that no image or attachment points at, and a walk that has already marked them seen never comes back for them. `find_unimported` lists those files; `queue_unimported` turns the same list into an import batch that attaches each file where it lies (`scan_mode="library"`), so a file the library already holds, or one the import filters reject, is removed instead. Only files under a folder named for an existing artist (matched by slug, so a differently-cased folder still matches) are considered, and only images, videos and archives. Files changed in the last day are left for the download that may still be writing them. """ from __future__ import annotations import os import stat import time from collections import defaultdict from dataclasses import dataclass from pathlib import Path from sqlalchemy import select from sqlalchemy.orm import Session from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source from ..utils.slug import slugify from ..utils.staging import is_temp_staging_file from .archive_extractor import is_archive from .importer import is_supported # Top-level folders that hold the library's own stores, not an artist's files. _STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"}) MIN_AGE_SECONDS = 24 * 3600 _ACTIVE_TASK_STATUSES = ("pending", "queued", "processing") _SAMPLES_PER_ARTIST = 3 _ARTISTS_LISTED = 50 @dataclass(frozen=True) class Unimported: path: Path artist_slug: str task_type: str # "media" or "archive", as ImportTask.task_type size: int def _referenced_paths(session: Session) -> set[str]: paths: set[str] = set() for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)): paths.add(p) if thumb: paths.add(thumb) paths.update(session.execute(select(PostAttachment.path)).scalars()) paths.update(session.execute( select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES)) ).scalars()) return paths def find_unimported( session: Session, images_root: Path, *, now: float | None = None, ) -> tuple[list[Unimported], dict]: """Walk the artists' folders and return (files to import, summary). The summary counts what was found and what was passed over, and lists the artists with the most to import (bytes first) with a few sample paths each: {"files", "bytes", "archives", "artists": [{slug, files, bytes, samples}], "recent", "not_media", "no_artist"}""" now = time.time() if now is None else now referenced = _referenced_paths(session) slugs = set(session.execute(select(Artist.slug)).scalars()) found: list[Unimported] = [] passed = {"recent": 0, "not_media": 0, "no_artist": 0} root = Path(images_root) for top in sorted(os.listdir(root)): if top in _STORE_DIRS or top.startswith(("_", ".")): continue top_dir = root / top if not top_dir.is_dir(): continue slug = slugify(top) known = slug in slugs for dirpath, dirnames, filenames in os.walk(top_dir): dirnames[:] = sorted(d for d in dirnames if not d.startswith(".")) for name in sorted(filenames): path = Path(dirpath) / name if ( name.startswith(".") or path.suffix.lower() == ".json" or is_temp_staging_file(path) ): continue if str(path) in referenced: continue try: st = path.lstat() except OSError: continue if not stat.S_ISREG(st.st_mode): continue kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None) if kind is None: passed["not_media"] += 1 elif now - st.st_mtime < MIN_AGE_SECONDS: passed["recent"] += 1 elif not known: passed["no_artist"] += 1 else: found.append(Unimported(path, slug, kind, st.st_size)) per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []}) for u in found: a = per_artist[u.artist_slug] a["files"] += 1 a["bytes"] += u.size if len(a["samples"]) < _SAMPLES_PER_ARTIST: a["samples"].append(str(u.path.relative_to(root))) artists = sorted( ({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"], )[:_ARTISTS_LISTED] summary = { "files": len(found), "bytes": sum(u.size for u in found), "archives": sum(1 for u in found if u.task_type == "archive"), "artists": artists, **passed, } return found, summary def queue_unimported( session: Session, images_root: Path, *, triggered_by: str = "manual", ) -> dict: """Create a `library` import batch holding every file `find_unimported` lists, and queue it. Returns the same summary plus `batch_id` (None when there was nothing to import).""" from ..tasks.import_file import enqueue_import # tasks import services found, summary = find_unimported(session, images_root) summary["batch_id"] = None if not found: return summary batch = ImportBatch( triggered_by=triggered_by, source_path=str(images_root), scan_mode="library", status="running", total_files=len(found), ) session.add(batch) session.flush() tasks = [ ImportTask( batch_id=batch.id, source_path=str(u.path), task_type=u.task_type, status="queued", size_bytes=u.size, ) for u in found ] session.add_all(tasks) session.commit() for t in tasks: enqueue_import(t.id, t.task_type) summary["batch_id"] = batch.id return summary def folder_owner( session: Session, path: Path, images_root: Path, ) -> tuple[Artist | None, Source | None]: """The artist whose folder holds `path`, and that artist's source for the platform folder below it when there is exactly one (a Discord artist has a source per channel, so there the post's sidecar decides).""" parts = Path(path).relative_to(images_root).parts artist = session.execute( select(Artist).where(Artist.slug == slugify(parts[0])) ).scalar_one_or_none() if artist is None or len(parts) < 3: return artist, None sources = session.execute( select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1]) ).scalars().all() return artist, (sources[0] if len(sources) == 1 else None)