feat: Cleanup can import files that are on disk but not in the library

A download writes its files into the artist's folder and imports them in a
later step. A run that stopped in between left complete files no record
points at, and a walk that had already marked them seen never returned for
them; on the live instance that is about 95 GB of art.

"Import files the library is missing" (Cleanup, Duplicates & leftovers)
walks every folder named for an artist, matched by slug so a differently
cased folder still matches, and lists the images, videos and archives no
image, attachment or queued import points at. Stores, sidecars, staging
files and files changed in the last day are left alone. Preview shows the
counts per artist; Import queues the same list as a library import batch,
whose tasks file the folder's post record, attach each file where it lies
with its folder's artist and source, and remove the ones the library
already holds or the import filters reject. A file whose import crashed is
kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/tasks/admin.py

@@ -459,3 +459,29 @@ def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
         return cleanup_service.reclaim_orphaned_attachments(
             session, images_root=IMAGES_ROOT, dry_run=dry_run,
         )
+
+
+@celery.task(
+    name="backend.app.tasks.admin.import_unimported_files_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    # One walk of the artists' folders: a stat per file, so minutes on a large
+    # library on network storage. The limits are the backstop for a wedged
+    # filesystem, not the expected exit.
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def import_unimported_files_task(self, dry_run: bool = True) -> dict:
+    """Find files in the artists' folders that the library has no record of.
+    dry_run (the default) returns what was found, per artist, without changing
+    anything; apply queues them as a `library` import batch, which attaches each
+    file where it lies and removes the ones the library already holds or the
+    import filters reject."""
+    from ..services import library_scan
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        if dry_run:
+            _found, summary = library_scan.find_unimported(session, IMAGES_ROOT)
+            return {**summary, "batch_id": None}
+        return library_scan.queue_unimported(session, IMAGES_ROOT)

backend/app/tasks/import_file.py

@@ -19,7 +19,8 @@ from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..models import ImportBatch, ImportSettings, ImportTask
-from ..services.importer import Importer
+from ..services.importer import Importer, ImportResult
+from ..services.library_scan import folder_owner
 from ..services.thumbnailer import Thumbnailer
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -165,6 +166,27 @@ def enqueue_import(task_id: int, task_type: str) -> None:
         import_media_file.delay(task_id)
 
 
+def _attach_where_it_lies(session, importer: Importer, path: Path) -> ImportResult:
+    """Import a file that already sits in its artist's folder (a `library`
+    batch, from services.library_scan). It is attached in place with that
+    artist and source. The post record its folder holds (`_post.json`) is filed
+    first, so the post gets its title and date. Afterwards the file is removed
+    if the library didn't keep it: a copy of something it already holds, or a
+    file the import filters reject. A file whose import crashed stays, with the
+    task marked failed: it may be the only copy of an archive the extractor
+    can't read yet."""
+    if not path.is_file():
+        return ImportResult(status="skipped", error="no longer on disk")
+    artist, source = folder_owner(session, path, IMAGES_ROOT)
+    record = path.parent / "_post.json"
+    if artist is not None and record.is_file():
+        importer.upsert_post_record(record, artist=artist, source=source)
+    result = importer.attach_in_place(path, artist=artist, source=source)
+    if result.status != "failed":
+        importer.discard_after_attach(path, result)
+    return result
+
+
 def _do_import(session, task, import_task_id: int) -> dict:
     """Actual work, called from inside the resilience wrapper."""
     settings = ImportSettings.load_sync(session)
@@ -180,7 +202,10 @@ def _do_import(session, task, import_task_id: int) -> dict:
         deep=deep,
     )
 
-    result = importer.import_one(Path(task.source_path))
+    if batch is not None and batch.scan_mode == "library":
+        result = _attach_where_it_lies(session, importer, Path(task.source_path))
+    else:
+        result = importer.import_one(Path(task.source_path))
 
     if result.status in ("imported", "superseded"):
         task.status = "complete"

backend/app/tasks/maintenance.py

@@ -189,6 +189,9 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # external-fetch entry above — without an override a healthy in-flight walk
     # is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
     "backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
+    # Importing what's on disk stats every file in the artists' folders, and
+    # its hard limit is 40 min for the same wedged-filesystem case. 45 = 40 + 5.
+    "backend.app.tasks.admin.import_unimported_files_task": 45,
     # Head training and the manual head apply run to 65 min (3900s) — past the
     # ml queue's threshold (#4432). 70 = 65 + 5.
     "backend.app.tasks.ml.train_heads": 70,