FabledCurator/backend/app/services/library_scan.py

"""Files in the artists' folders that the library has no record of.

A download writes its files into the artist's folder and imports them in a
later step. A run that stops between the two (a restart, a time limit) leaves
complete files on disk that no image or attachment points at, and a walk that
has already marked them seen never comes back for them. `find_unimported`
lists those files; `queue_unimported` turns the same list into an import batch
that attaches each file where it lies (`scan_mode="library"`), so a file the
library already holds, or one the import filters reject, is removed instead.

Only files under a folder named for an existing artist (matched by slug, so a
differently-cased folder still matches) are considered, and only images,
videos and archives. Files changed in the last day are left for the download
that may still be writing them.
"""
from __future__ import annotations

import os
import stat
import time
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path

from sqlalchemy import select
from sqlalchemy.orm import Session

from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source
from ..utils.slug import slugify
from ..utils.staging import is_temp_staging_file
from .archive_extractor import is_archive
from .importer import is_supported

# Top-level folders that hold the library's own stores, not an artist's files.
_STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"})
MIN_AGE_SECONDS = 24 * 3600
_ACTIVE_TASK_STATUSES = ("pending", "queued", "processing")
_SAMPLES_PER_ARTIST = 3
_ARTISTS_LISTED = 50


@dataclass(frozen=True)
class Unimported:
    path: Path
    artist_slug: str
    task_type: str  # "media" or "archive", as ImportTask.task_type
    size: int


def _referenced_paths(session: Session) -> set[str]:
    paths: set[str] = set()
    for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)):
        paths.add(p)
        if thumb:
            paths.add(thumb)
    paths.update(session.execute(select(PostAttachment.path)).scalars())
    paths.update(session.execute(
        select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES))
    ).scalars())
    return paths


def find_unimported(
    session: Session, images_root: Path, *, now: float | None = None,
) -> tuple[list[Unimported], dict]:
    """Walk the artists' folders and return (files to import, summary).

    The summary counts what was found and what was passed over, and lists the
    artists with the most to import (bytes first) with a few sample paths each:
        {"files", "bytes", "archives", "artists": [{slug, files, bytes,
         samples}], "recent", "not_media", "no_artist"}"""
    now = time.time() if now is None else now
    referenced = _referenced_paths(session)
    slugs = set(session.execute(select(Artist.slug)).scalars())
    found: list[Unimported] = []
    passed = {"recent": 0, "not_media": 0, "no_artist": 0}
    root = Path(images_root)
    for top in sorted(os.listdir(root)):
        if top in _STORE_DIRS or top.startswith(("_", ".")):
            continue
        top_dir = root / top
        if not top_dir.is_dir():
            continue
        slug = slugify(top)
        known = slug in slugs
        for dirpath, dirnames, filenames in os.walk(top_dir):
            dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
            for name in sorted(filenames):
                path = Path(dirpath) / name
                if (
                    name.startswith(".") or path.suffix.lower() == ".json"
                    or is_temp_staging_file(path)
                ):
                    continue
                if str(path) in referenced:
                    continue
                try:
                    st = path.lstat()
                except OSError:
                    continue
                if not stat.S_ISREG(st.st_mode):
                    continue
                kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None)
                if kind is None:
                    passed["not_media"] += 1
                elif now - st.st_mtime < MIN_AGE_SECONDS:
                    passed["recent"] += 1
                elif not known:
                    passed["no_artist"] += 1
                else:
                    found.append(Unimported(path, slug, kind, st.st_size))

    per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []})
    for u in found:
        a = per_artist[u.artist_slug]
        a["files"] += 1
        a["bytes"] += u.size
        if len(a["samples"]) < _SAMPLES_PER_ARTIST:
            a["samples"].append(str(u.path.relative_to(root)))
    artists = sorted(
        ({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"],
    )[:_ARTISTS_LISTED]
    summary = {
        "files": len(found),
        "bytes": sum(u.size for u in found),
        "archives": sum(1 for u in found if u.task_type == "archive"),
        "artists": artists,
        **passed,
    }
    return found, summary


def queue_unimported(
    session: Session, images_root: Path, *, triggered_by: str = "manual",
) -> dict:
    """Create a `library` import batch holding every file `find_unimported`
    lists, and queue it. Returns the same summary plus `batch_id` (None when
    there was nothing to import)."""
    from ..tasks.import_file import enqueue_import  # tasks import services

    found, summary = find_unimported(session, images_root)
    summary["batch_id"] = None
    if not found:
        return summary
    batch = ImportBatch(
        triggered_by=triggered_by, source_path=str(images_root),
        scan_mode="library", status="running", total_files=len(found),
    )
    session.add(batch)
    session.flush()
    tasks = [
        ImportTask(
            batch_id=batch.id, source_path=str(u.path), task_type=u.task_type,
            status="queued", size_bytes=u.size,
        )
        for u in found
    ]
    session.add_all(tasks)
    session.commit()
    for t in tasks:
        enqueue_import(t.id, t.task_type)
    summary["batch_id"] = batch.id
    return summary


def folder_owner(
    session: Session, path: Path, images_root: Path,
) -> tuple[Artist | None, Source | None]:
    """The artist whose folder holds `path`, and that artist's source for the
    platform folder below it when there is exactly one (a Discord artist has a
    source per channel, so there the post's sidecar decides)."""
    parts = Path(path).relative_to(images_root).parts
    artist = session.execute(
        select(Artist).where(Artist.slug == slugify(parts[0]))
    ).scalar_one_or_none()
    if artist is None or len(parts) < 3:
        return artist, None
    sources = session.execute(
        select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1])
    ).scalars().all()
    return artist, (sources[0] if len(sources) == 1 else None)