feat(external): file-host fetcher subsystem (Phase 4a)

Shared, reusable fetchers for the 5 off-platform hosts behind one signature (fetch_external(host, url, dest_dir, ...) -> FetchResult): - dropbox : force dl=1 + stream GET - pixeldrain : GET /api/file/{id} - mediafire : scrape the download page for the direct link + stream GET - gdrive : gdown (confirm-token + virus-scan interstitial); added to reqs - mega : MEGAcmd `mega-get` subprocess (public link incl. #key) HTTP/gdown/subprocess go through module seams so unit tests run without network/gdown/MEGAcmd. fetch_external never raises — every backend failure (transport, non-200, scrape miss, subprocess error, stop) is captured on .error so the worker (next slice) records it and moves on. mega's binary lands in the runtime image in a later slice; the code is complete + tested now. Refs FC #830 (Phase 4a). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-14 13:29:20 -04:00
parent 896e4f248c
commit 13253b18d1
3 changed files with 393 additions and 0 deletions
@@ -0,0 +1,141 @@
+"""Unit tests for external_fetch — no network/gdown/MEGAcmd (seams stubbed)."""
+
+from pathlib import Path
+
+import pytest
+import requests
+
+import backend.app.services.external_fetch as ef
+from backend.app.services.external_fetch import fetch_external
+
+
+def _resp(*, status=200, headers=None, chunks=(b"hello",), text=""):
+    class _R:
+        def __init__(self):
+            self.status_code = status
+            self.headers = headers or {}
+            self.text = text
+
+        def iter_content(self, chunk_size=65536):
+            yield from chunks
+
+    return _R()
+
+
+def test_dropbox_forces_dl1_and_downloads(tmp_path, monkeypatch):
+    seen = {}
+
+    def fake_get(url, *, timeout, headers=None, stream=True):
+        seen["url"] = url
+        return _resp(headers={"Content-Disposition": 'attachment; filename="film.zip"'})
+
+    monkeypatch.setattr(ef, "_http_get", fake_get)
+    res = fetch_external(
+        "dropbox", "https://www.dropbox.com/s/abc/film.zip?dl=0", tmp_path
+    )
+    assert res.ok
+    assert "dl=1" in seen["url"]
+    assert res.files[0].name == "film.zip"
+    assert res.files[0].read_bytes() == b"hello"
+
+
+def test_pixeldrain_hits_api_endpoint(tmp_path, monkeypatch):
+    seen = {}
+
+    def fake_get(url, *, timeout, headers=None, stream=True):
+        seen["url"] = url
+        return _resp()
+
+    monkeypatch.setattr(ef, "_http_get", fake_get)
+    res = fetch_external("pixeldrain", "https://pixeldrain.com/u/ems4UomN", tmp_path)
+    assert res.ok
+    assert seen["url"] == "https://pixeldrain.com/api/file/ems4UomN"
+
+
+def test_mediafire_scrapes_then_downloads(tmp_path, monkeypatch):
+    calls = []
+
+    def fake_get(url, *, timeout, headers=None, stream=True):
+        calls.append(url)
+        if "mediafire.com/file" in url:
+            return _resp(
+                text='<a id="downloadButton" '
+                'href="https://download1234.mediafire.com/abc/film.zip">dl</a>'
+            )
+        return _resp(headers={"Content-Disposition": 'filename="film.zip"'})
+
+    monkeypatch.setattr(ef, "_http_get", fake_get)
+    res = fetch_external("mediafire", "https://www.mediafire.com/file/x/film.zip", tmp_path)
+    assert res.ok
+    assert res.files[0].name == "film.zip"
+    assert calls[1].startswith("https://download1234.mediafire.com/")
+
+
+def test_mediafire_missing_link_errors(tmp_path, monkeypatch):
+    monkeypatch.setattr(ef, "_http_get", lambda *a, **k: _resp(text="<html>none</html>"))
+    res = fetch_external("mediafire", "https://www.mediafire.com/file/x", tmp_path)
+    assert not res.ok
+    assert "not found" in res.error
+
+
+def test_gdrive_uses_gdown(tmp_path, monkeypatch):
+    def fake_gdown(url, out_dir):
+        p = Path(out_dir) / "drive.bin"
+        p.write_bytes(b"xyz")
+        return str(p)
+
+    monkeypatch.setattr(ef, "_gdown_download", fake_gdown)
+    res = fetch_external("gdrive", "https://drive.google.com/file/d/ID/view", tmp_path)
+    assert res.ok
+    assert res.files[0].name == "drive.bin"
+    assert res.bytes == 3
+
+
+def test_gdrive_no_file_errors(tmp_path, monkeypatch):
+    monkeypatch.setattr(ef, "_gdown_download", lambda url, out_dir: None)
+    res = fetch_external("gdrive", "https://drive.google.com/file/d/ID/view", tmp_path)
+    assert not res.ok
+    assert "gdown" in res.error
+
+
+def test_mega_collects_new_files(tmp_path, monkeypatch):
+    def fake_mega(url, out_dir, *, timeout):
+        (Path(out_dir) / "film.mp4").write_bytes(b"vid")
+
+    monkeypatch.setattr(ef, "_run_mega_get", fake_mega)
+    res = fetch_external("mega", "https://mega.nz/file/x#k", tmp_path)
+    assert res.ok
+    assert res.files[0].name == "film.mp4"
+
+
+def test_unsupported_host(tmp_path):
+    res = fetch_external("nope", "https://x", tmp_path)
+    assert not res.ok
+    assert "unsupported" in res.error
+
+
+def test_non_200_errors(tmp_path, monkeypatch):
+    monkeypatch.setattr(ef, "_http_get", lambda *a, **k: _resp(status=404))
+    res = fetch_external("pixeldrain", "https://pixeldrain.com/u/x", tmp_path)
+    assert not res.ok
+    assert "404" in res.error
+
+
+def test_transport_error_captured(tmp_path, monkeypatch):
+    def boom(*a, **k):
+        raise requests.ConnectionError("down")
+
+    monkeypatch.setattr(ef, "_http_get", boom)
+    res = fetch_external("dropbox", "https://www.dropbox.com/s/x?dl=0", tmp_path)
+    assert not res.ok
+    assert "transport" in res.error
+
+
+def test_should_stop_aborts_and_cleans_part(tmp_path, monkeypatch):
+    monkeypatch.setattr(ef, "_http_get", lambda *a, **k: _resp(chunks=(b"a", b"b")))
+    res = fetch_external(
+        "pixeldrain", "https://pixeldrain.com/u/x", tmp_path,
+        should_stop=lambda: True,
+    )
+    assert not res.ok
+    assert list(tmp_path.glob("*.part")) == []