fix: a Mega folder link's files in subfolders are imported, and a retry no longer deletes complete files

_fetch_mega listed only the top level of the link dir, so a folder link's
files in subfolders never reached the importer: the link either failed with
"wrote no new file" or read downloaded with its subfolders left on disk.
It now walks the tree.

The worker cleared every unrecorded file in a link dir before each fetch
and after a failed one. Combined with the daily recovery of dead links,
that deletes complete downloads whose upstream is gone, which may be the
only copy. Every fetcher stages to .part or .megatmp.* and renames on
completion, so the clear now removes staging files only, and a successful
fetch imports every complete unrecorded file in the link dir, including
ones an earlier attempt finished.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_external_worker.py

@@ -377,6 +377,74 @@ def test_a_rejected_download_is_removed(db_sync, tmp_path, monkeypatch):
     assert db_sync.get(ExternalLink, link_id).status == "downloaded"
 
 
+def _other_jpeg_bytes():
+    # A second structured image, unlike _structured_jpeg_bytes, so the two
+    # don't near-duplicate each other at import.
+    im = Image.new("RGB", (256, 256))
+    px = im.load()
+    for y in range(256):
+        for x in range(256):
+            px[x, y] = ((x * x) % 256, (y * 3) % 256, (x ^ y) % 256)
+    buf = io.BytesIO()
+    im.save(buf, "JPEG")
+    return buf.getvalue()
+
+
+def test_a_failed_fetch_keeps_complete_files(db_sync, tmp_path, monkeypatch):
+    """A failed fetch removes staging files only. A complete file may be the
+    only copy left once the host has dropped the upload."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        (dest_dir / "images").mkdir(parents=True, exist_ok=True)
+        (dest_dir / "images" / "done.jpg").write_bytes(_structured_jpeg_bytes())
+        (dest_dir / "images" / ".megatmp.AbCd").write_bytes(b"half")
+        return FetchResult(error="API call 'f' failed: ENOENT")
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    ext.fetch_external_link(link_id)
+
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    assert (link_dir / "images" / "done.jpg").exists()
+    assert not (link_dir / "images" / ".megatmp.AbCd").exists()
+
+
+def test_a_successful_fetch_imports_what_an_earlier_attempt_finished(
+    db_sync, tmp_path, monkeypatch,
+):
+    """Complete files an earlier attempt wrote, in subfolders too, are imported
+    with the new fetch's files instead of being deleted or left behind."""
+    _, link = _seed(db_sync, host="mega")
+    link_id = link.id
+    monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
+    link_dir = _link_dir(tmp_path, "mega", link_id)
+    (link_dir / "pack").mkdir(parents=True)
+    earlier = link_dir / "pack" / "earlier.jpg"
+    earlier.write_bytes(_structured_jpeg_bytes())
+
+    def fake_fetch(host, url, dest_dir, **kwargs):
+        f = dest_dir / "new.jpg"
+        f.write_bytes(_other_jpeg_bytes())
+        return FetchResult(files=[f], bytes=f.stat().st_size)
+
+    monkeypatch.setattr(ext, "fetch_external", fake_fetch)
+    import backend.app.tasks.ml as ml_mod
+    import backend.app.tasks.thumbnail as thumb_mod
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
+    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
+
+    out = ext.fetch_external_link(link_id)
+
+    assert out["images"] == 2
+    db_sync.expire_all()
+    paths = set(db_sync.execute(select(ImageRecord.path)).scalars())
+    assert {str(earlier), str(link_dir / "new.jpg")} <= paths
+
+
 def test_sweep_recovers_a_link_whose_worker_died(db_sync, monkeypatch):
     """A link 'downloading' past the task's hard limit (or with no claim time)
     counts as a failed attempt and is retried; a fresh claim is left alone."""