fix: a Mega folder link's files in subfolders are imported, and a retry no longer deletes complete files

_fetch_mega listed only the top level of the link dir, so a folder link's
files in subfolders never reached the importer: the link either failed with
"wrote no new file" or read downloaded with its subfolders left on disk.
It now walks the tree.

The worker cleared every unrecorded file in a link dir before each fetch
and after a failed one. Combined with the daily recovery of dead links,
that deletes complete downloads whose upstream is gone, which may be the
only copy. Every fetcher stages to .part or .megatmp.* and renames on
completion, so the clear now removes staging files only, and a successful
fetch imports every complete unrecorded file in the link dir, including
ones an earlier attempt finished.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/external_fetch.py

@@ -241,11 +241,19 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
 def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
                 total_timeout: float,
                 should_stop: Callable[[], bool]) -> FetchResult:
-    before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
+    # A folder link keeps its subfolders, so files are found at any depth.
+    # Dotfiles are megatools' `.megatmp.*` staging names.
+    def _complete_files() -> set[Path]:
+        return {
+            p for p in dest_dir.rglob("*")
+            if p.is_file() and not p.name.startswith(".")
+        } if dest_dir.exists() else set()
+
+    before = _complete_files()
     # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
     # timeout has no analogue here), so the total budget applies directly.
     _run_mega_get(url, str(dest_dir), timeout=total_timeout)
-    new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
+    new = sorted(_complete_files() - before)
     if not new:
         return FetchResult(error="mega-get wrote no new file")
     return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))

backend/app/tasks/external.py

@@ -33,6 +33,7 @@ from ..services.external_fetch import fetch_external
 from ..services.importer import Importer
 from ..services.thumbnailer import Thumbnailer
 from ._sync_engine import sync_session_factory as _sync_session_factory
+from .maintenance import is_temp_staging_file
 
 log = logging.getLogger(__name__)
 
@@ -189,10 +190,9 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
         / str(post.external_post_id) / str(link_id)
     )
     try:
-        # Clear what an earlier, interrupted attempt left behind, so this fetch
-        # starts from the link's imported files only.
-        with SessionLocal() as session:
-            _discard_unrecorded(session, post_dir)
+        # Clear the staging files an interrupted attempt left behind. Complete
+        # files it left stay, and are imported once this fetch succeeds.
+        _discard_partials(post_dir)
         result = fetch_external(host, url, post_dir)
         with SessionLocal() as session:
             link = session.get(ExternalLink, link_id)
@@ -203,7 +203,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
                 )
                 _record_failure(link, attempts, result.error or "fetch failed")
                 session.commit()
-                _discard_unrecorded(session, post_dir)
+                _discard_partials(post_dir)
                 return {"link_id": link_id, "error": result.error}
 
             log.info(
@@ -215,7 +215,8 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             post = session.get(Post, link.post_id)
             artist = session.get(Artist, post.artist_id)
             source = session.get(Source, post.source_id) if post.source_id else None
-            image_ids = _route_files(session, result.files, post, platform, artist, source)
+            files = _unimported_files(session, post_dir, first=result.files)
+            image_ids = _route_files(session, files, post, platform, artist, source)
             link.status = "downloaded"
             link.last_error = None
             link.completed_at = datetime.now(UTC)
@@ -245,7 +246,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
             if link is not None and link.status == "downloading":
                 _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
                 session.commit()
-            _discard_unrecorded(session, post_dir)
+        _discard_partials(post_dir)
         raise
     finally:
         if lock is not None:
@@ -334,25 +335,37 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
     return image_ids
 
 
-def _discard_unrecorded(session, link_dir: Path) -> int:
-    """Delete every file in a link's staging dir that no ImageRecord points at,
-    then the dirs left empty. The dir belongs to one link, so anything in it
-    without a record is a partial or abandoned download from an earlier
-    attempt (megatools' `.megatmp.*`, a killed transfer, a rejected file).
-    Images imported in place stay. Returns the number of files removed."""
+def _unimported_files(session, link_dir: Path, *, first=()) -> list[Path]:
+    """Every complete file under a link's dir, subfolders included, that no
+    ImageRecord points at: what this fetch wrote plus anything an earlier
+    attempt finished writing but never imported. `first` (the fetch's own
+    files) leads the list. Staging files and sidecars are left out."""
     if not link_dir.is_dir():
-        return 0
-    files = [p for p in link_dir.rglob("*") if p.is_file()]
-    if not files:
-        return 0
+        return []
+    found = {
+        p for p in link_dir.rglob("*")
+        if p.is_file() and not is_temp_staging_file(p) and p.suffix.lower() != ".json"
+    }
+    ordered = [p for p in first if p in found] + sorted(found.difference(first))
     recorded = set(session.execute(
         select(ImageRecord.path).where(
-            ImageRecord.path.in_([str(p) for p in files])
+            ImageRecord.path.in_([str(p) for p in ordered])
         )
-    ).scalars())
+    ).scalars()) if ordered else set()
+    return [p for p in ordered if str(p) not in recorded]
+
+
+def _discard_partials(link_dir: Path) -> int:
+    """Delete the staging files under a link's dir (megatools' `.megatmp.*`, an
+    HTTP fetch's `.part`), then the dirs left empty. Every fetcher writes to a
+    staging name and renames on completion, so a staging file is a transfer
+    that never finished. Complete files stay: they may be the only copy of art
+    whose upstream has since gone. Returns the number of files removed."""
+    if not link_dir.is_dir():
+        return 0
     removed = 0
-    for p in files:
-        if str(p) in recorded:
+    for p in link_dir.rglob("*"):
+        if not (p.is_file() and is_temp_staging_file(p)):
             continue
         try:
             p.unlink(missing_ok=True)
@@ -363,7 +376,7 @@ def _discard_unrecorded(session, link_dir: Path) -> int:
         try:
             d.rmdir()
         except OSError:
-            pass  # not empty: it holds an imported file
+            pass  # not empty: it holds a complete file
     try:
         link_dir.rmdir()
     except OSError:
@@ -381,7 +394,8 @@ def sweep_external_links() -> dict:
     First it recovers links whose worker died mid-fetch: a link 'downloading'
     for longer than the task's hard limit has no worker left, so it counts as a
     failed attempt and goes back to the retry path (dead-lettered at the
-    threshold like any other failure). The next attempt clears its leftovers."""
+    threshold like any other failure). The next attempt clears its staging
+    files and imports the complete ones."""
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
         now = datetime.now(UTC)