FabledCurator/tests/test_sidecar_import.py

"""FC-2d-v: sidecar → Source/Post/ImageProvenance (sync importer)."""

import json
from pathlib import Path

import pytest
from PIL import Image
from sqlalchemy import func, select

from backend.app.models import (
    Artist,
    ExternalLink,
    ImageProvenance,
    ImageRecord,
    ImportSettings,
    Post,
    Source,
)
from backend.app.services.importer import Importer
from backend.app.services.thumbnailer import Thumbnailer

pytestmark = pytest.mark.integration


@pytest.fixture
def import_layout(tmp_path):
    import_root = tmp_path / "import"
    images_root = tmp_path / "images"
    import_root.mkdir()
    images_root.mkdir()
    return import_root, images_root


@pytest.fixture
def importer(db_sync, import_layout):
    import_root, images_root = import_layout
    settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    return Importer(
        session=db_sync,
        images_root=images_root,
        import_root=import_root,
        thumbnailer=Thumbnailer(images_root=images_root),
        settings=settings,
    )


def _split(path: Path, orient, size=(256, 256)):
    """Structured image (half/half) — solid colors phash-collapse, which
    would route a 2nd image to the supersede path (no sidecar applied)."""
    path.parent.mkdir(parents=True, exist_ok=True)
    w, h = size
    im = Image.new("L", size, 0)
    px = im.load()
    for y in range(h):
        for x in range(w):
            if (x / w if orient == "v" else y / h) >= 0.5:
                px[x, y] = 255
    im.convert("RGB").save(path, "JPEG")


def _sidecar(media: Path, payload: dict):
    media.with_suffix(".json").write_text(json.dumps(payload))


def test_sidecar_creates_provenance(importer, import_layout):
    import_root, _ = import_layout
    m = import_root / "Alice" / "a.jpg"
    _split(m, "v")
    _sidecar(m, {
        "category": "patreon", "id": 555,
        "url": "https://patreon.com/posts/555", "title": "Set 1",
        "content": "<p>hi</p>", "page_count": 2,
        "published_at": "2023-08-01T00:00:00Z",
    })
    r = importer.import_one(m)
    assert r.status == "imported"
    rec = importer.session.get(ImageRecord, r.image_id)
    post = importer.session.execute(select(Post)).scalar_one()
    # Filesystem-imported sidecar posts no longer create a synthetic Source
    # (alembic 0030 / nullable post.source_id refactor). The Post is linked
    # to the artist via Post.artist_id; Post.source_id stays NULL until a
    # real subscription for the (artist, platform) gets added.
    assert post.source_id is None
    assert importer.session.execute(
        select(func.count()).select_from(Source)
    ).scalar_one() == 0
    assert post.external_post_id == "555"
    assert post.post_url == "https://patreon.com/posts/555"
    assert post.post_title == "Set 1"
    assert post.description == "<p>hi</p>"
    assert post.attachment_count == 2
    assert post.post_date is not None
    assert post.raw_metadata["id"] == 555
    prov = importer.session.execute(select(ImageProvenance)).scalar_one()
    assert prov.image_record_id == rec.id and prov.post_id == post.id
    assert rec.primary_post_id == post.id
    # Denormalized gallery sort key (alembic 0035) tracks the primary post's
    # publish date so /scroll orders off ix_image_record_effective_date.
    assert rec.effective_date == post.post_date


def test_sidecar_source_url_persists_filehash(importer, import_layout):
    """#830 Phase 2: a media sidecar's source_url lands on the ImageRecord as
    source_url + the lowercased CDN filehash (the inline-image join key)."""
    import_root, _ = import_layout
    m = import_root / "Alice" / "b.jpg"
    _split(m, "v")
    _sidecar(m, {
        "category": "patreon", "id": 901, "title": "Body post",
        "source_url": "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg",
    })
    r = importer.import_one(m)
    assert r.status == "imported"
    rec = importer.session.get(ImageRecord, r.image_id)
    assert rec.source_url == (
        "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg"
    )
    assert rec.source_filehash == "0123456789abcdef0123456789abcdef"


def test_relink_source_filehash_backfills_existing(importer, import_layout):
    """#830 recapture (Part B): relink_source_filehash backfills source_url +
    source_filehash on an existing on-disk image's ImageRecord (matched by
    sha256) WITHOUT re-importing or unlinking the file. NULL-only — never
    clobbers a filehash already set."""
    import_root, _ = import_layout
    m = import_root / "Alice" / "img.jpg"
    _split(m, "v")
    r = importer.import_one(m)
    assert r.status == "imported"
    rec = importer.session.get(ImageRecord, r.image_id)
    assert rec.source_filehash is None

    url = "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/img.jpg"
    assert importer.relink_source_filehash(m, url) is True
    importer.session.refresh(rec)
    assert rec.source_url == url
    assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
    assert m.exists()  # never unlinked

    # NULL-only: a second relink with a different url is a no-op.
    other = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/x.jpg"
    assert importer.relink_source_filehash(m, other) is False
    importer.session.refresh(rec)
    assert rec.source_filehash == "0123456789abcdef0123456789abcdef"


def test_relink_source_filehash_no_match_is_noop(importer, import_layout):
    """A path whose bytes match no ImageRecord (never imported) is a clean no-op,
    not an error."""
    import_root, _ = import_layout
    m = import_root / "Alice" / "ghost.jpg"
    _split(m, "h")
    assert importer.relink_source_filehash(
        m, "https://cdn.test/p/0123456789abcdef0123456789abcdef/ghost.jpg"
    ) is False


def test_link_existing_image_to_post_backfills_provenance(importer, import_layout, db_sync):
    """#1288: an on-disk image imported BARE (no sidecar → no post link, as the
    pre-cutover gallery-dl pixiv images were) gets linked to its post from the
    recapture walk's (path, external_post_id) pairing — image_provenance +
    primary_post_id — without re-importing. Idempotent."""
    import_root, _ = import_layout
    m = import_root / "Alice" / "img.jpg"
    _split(m, "v")
    rec = importer.session.get(ImageRecord, importer.import_one(m).image_id)
    assert rec.primary_post_id is None          # bare: no post link yet
    artist = db_sync.get(Artist, rec.artist_id)
    src = Source(
        artist_id=artist.id, platform="pixiv",
        url="https://www.pixiv.net/users/1", enabled=True,
    )
    db_sync.add(src)
    db_sync.flush()
    # The post already exists (upsert_post_record wrote it earlier in recapture).
    post = Post(
        source_id=src.id, artist_id=artist.id,
        external_post_id="146132304", post_title="t",
    )
    db_sync.add(post)
    db_sync.commit()

    assert importer.link_existing_image_to_post(
        Path(rec.path), "146132304", source=src, artist=artist,
    ) is True
    importer.session.refresh(rec)
    assert rec.primary_post_id == post.id
    prov = db_sync.execute(select(ImageProvenance).where(
        ImageProvenance.image_record_id == rec.id,
        ImageProvenance.post_id == post.id,
    )).scalar_one()
    assert prov.source_id == src.id

    # Idempotent: a second call re-affirms without duplicating the row.
    assert importer.link_existing_image_to_post(
        Path(rec.path), "146132304", source=src, artist=artist,
    ) is True
    n = db_sync.execute(select(func.count(ImageProvenance.id)).where(
        ImageProvenance.image_record_id == rec.id,
        ImageProvenance.post_id == post.id,
    )).scalar_one()
    assert n == 1


def test_link_existing_image_to_post_no_match_is_noop(importer):
    """A path with no ImageRecord, or an empty external id, is a clean no-op."""
    assert importer.link_existing_image_to_post(
        Path("/nonexistent/x.jpg"), "999",
    ) is False


def test_reimport_same_post_idempotent(importer, import_layout):
    import_root, _ = import_layout
    # Threshold 0: only an exact phash match collapses. Orthogonal splits
    # still sit within the default Hamming-10 at 64-bit, so without this
    # the 2nd image would be skipped as a near-dup (phash dedup working).
    importer.settings.phash_threshold = 0
    payload = {"category": "patreon", "id": 777, "title": "P"}
    m1 = import_root / "Bob" / "p1.jpg"
    _split(m1, "v")
    _sidecar(m1, payload)
    importer.import_one(m1)
    m2 = import_root / "Bob" / "p2.jpg"
    _split(m2, "h")            # distinct phash; threshold 0 → both import
    _sidecar(m2, payload)
    r2 = importer.import_one(m2)
    assert r2.status == "imported"
    # No synthetic Source after alembic 0030; both imports still resolve to
    # a single null-source Post (deduped by uq_post_artist_external_id_null_source).
    assert importer.session.execute(
        select(func.count()).select_from(Source)
    ).scalar_one() == 0
    assert importer.session.execute(
        select(func.count()).select_from(Post)
    ).scalar_one() == 1
    assert importer.session.execute(
        select(func.count()).select_from(ImageProvenance)
    ).scalar_one() == 2


def test_garbage_sidecar_still_imports(importer, import_layout):
    import_root, _ = import_layout
    m = import_root / "Carol" / "c.jpg"
    _split(m, "v")
    m.with_suffix(".json").write_text("{ not json")
    r = importer.import_one(m)
    assert r.status == "imported"
    assert importer.session.execute(
        select(func.count()).select_from(Post)
    ).scalar_one() == 0


def test_no_sidecar_unchanged(importer, import_layout):
    import_root, _ = import_layout
    m = import_root / "Dave" / "d.jpg"
    _split(m, "v")
    r = importer.import_one(m)
    assert r.status == "imported"
    assert importer.session.execute(
        select(func.count()).select_from(Post)
    ).scalar_one() == 0


def test_no_artist_anywhere_skips_provenance(importer, import_layout):
    import_root, _ = import_layout
    m = import_root / "rootfile.jpg"          # no top-level artist folder
    _split(m, "v")
    _sidecar(m, {"category": "x", "id": 1})   # no artist key
    r = importer.import_one(m)
    assert r.status == "imported"
    assert importer.session.execute(
        select(func.count()).select_from(Source)
    ).scalar_one() == 0


def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
    import_root, _ = import_layout
    m = import_root / "e.jpg"                 # root → no folder artist
    _split(m, "v")
    _sidecar(m, {"category": "pixiv", "id": 9, "artist": "Yuki"})
    r = importer.import_one(m)
    assert r.status == "imported"
    a = importer.session.execute(
        select(Artist).where(Artist.slug == "yuki")
    ).scalar_one()
    # No synthetic Source after alembic 0030; the artist linkage lives on
    # Post.artist_id (NOT NULL FK).
    post = importer.session.execute(select(Post)).scalar_one()
    assert post.artist_id == a.id
    assert post.source_id is None
    assert importer.session.execute(
        select(func.count()).select_from(Source)
    ).scalar_one() == 0


def test_upsert_post_record_creates_media_less_post(importer, import_layout):
    """A post-only sidecar (no media) upserts a Post WITH its body — text posts
    are captured even when they have nothing to download."""
    import_root, _ = import_layout
    artist = Artist(name="Alice", slug="alice")
    importer.session.add(artist)
    importer.session.flush()
    sc = import_root / "Alice" / "_post.json"
    sc.parent.mkdir(parents=True, exist_ok=True)
    sc.write_text(json.dumps({
        "category": "patreon", "id": 777,
        "url": "https://patreon.com/posts/777", "title": "Text only",
        "content": "<p>links below: mega</p>",
        "published_at": "2023-08-01T00:00:00Z",
    }))
    assert importer.upsert_post_record(sc, artist=artist) is True
    post = importer.session.execute(select(Post)).scalar_one()
    assert post.external_post_id == "777"
    assert post.post_title == "Text only"
    assert post.description == "<p>links below: mega</p>"
    assert post.post_url == "https://patreon.com/posts/777"
    assert post.post_date is not None


def test_upsert_post_record_idempotent_no_double(importer, import_layout):
    """Re-running upsert_post_record UPDATES the same Post (keyed on
    external_post_id) — never doubles."""
    import_root, _ = import_layout
    artist = Artist(name="Bob", slug="bob")
    importer.session.add(artist)
    importer.session.flush()
    sc = import_root / "Bob" / "_post.json"
    sc.parent.mkdir(parents=True, exist_ok=True)
    sc.write_text(json.dumps({
        "category": "patreon", "id": 888, "title": "T",
        "content": "<p>body</p>", "url": "https://patreon.com/posts/888",
    }))
    assert importer.upsert_post_record(sc, artist=artist) is True
    assert importer.upsert_post_record(sc, artist=artist) is True
    assert importer.session.execute(
        select(func.count()).select_from(Post)
    ).scalar_one() == 1


def test_external_links_recorded_from_body(importer, import_layout):
    """A mega/gdrive/etc. link in the body is recorded as an external_link row
    (status pending) so it's never silently dropped."""
    import_root, _ = import_layout
    artist = Artist(name="Linker", slug="linker")
    importer.session.add(artist)
    importer.session.flush()
    sc = import_root / "Linker" / "_post.json"
    sc.parent.mkdir(parents=True, exist_ok=True)
    sc.write_text(json.dumps({
        "category": "patreon", "id": 999, "title": "Film",
        "url": "https://patreon.com/posts/999",
        "content": '<p>Get it: <a href="https://mega.nz/file/AbC#key123">Mega</a></p>',
    }))
    assert importer.upsert_post_record(sc, artist=artist) is True
    importer.session.expire_all()  # re-read so the server_default status loads
    links = importer.session.execute(select(ExternalLink)).scalars().all()
    assert len(links) == 1
    assert links[0].host == "mega"
    assert links[0].url == "https://mega.nz/file/AbC#key123"
    assert links[0].label == "Mega"
    assert links[0].status == "pending"


def test_external_links_not_duplicated_on_reimport(importer, import_layout):
    """Re-importing the same body keeps ONE external_link row (insert-missing)."""
    import_root, _ = import_layout
    artist = Artist(name="Linker2", slug="linker2")
    importer.session.add(artist)
    importer.session.flush()
    sc = import_root / "Linker2" / "_post.json"
    sc.parent.mkdir(parents=True, exist_ok=True)
    sc.write_text(json.dumps({
        "category": "patreon", "id": 1000, "title": "Film",
        "url": "https://patreon.com/posts/1000",
        "content": '<a href="https://pixeldrain.com/u/xyz">px</a>',
    }))
    assert importer.upsert_post_record(sc, artist=artist) is True
    assert importer.upsert_post_record(sc, artist=artist) is True
    assert importer.session.execute(
        select(func.count()).select_from(ExternalLink)
    ).scalar_one() == 1


def test_post_record_redates_images_linked_before_it(importer, import_layout):
    """#4431: the native ingesters import a message's media before its record,
    and only the record carries the date. The images start on their download
    time; when the record lands they take the post's date."""
    import_root, _ = import_layout
    artist = Artist(name="Alice", slug="alice")
    importer.session.add(artist)
    importer.session.flush()
    m = import_root / "Alice" / "20240301_123_01_art.jpg"
    _split(m, "v")
    _sidecar(m, {"category": "discord", "message_id": "123"})
    r = importer.import_one(m)
    assert r.status == "imported"
    rec = importer.session.get(ImageRecord, r.image_id)
    post = importer.session.execute(select(Post)).scalar_one()
    assert post.post_date is None
    download_time = rec.effective_date

    sc = import_root / "Alice" / "20240301_123_post.json"
    sc.write_text(json.dumps({
        "category": "discord", "message_id": "123", "message": "",
        "date": "2024-03-01T18:30:00.000000+00:00",
    }))
    assert importer.upsert_post_record(sc, artist=artist) is True
    importer.session.expire_all()
    rec = importer.session.get(ImageRecord, r.image_id)
    post = importer.session.execute(select(Post)).scalar_one()
    assert post.post_date is not None
    assert post.post_date != download_time
    assert rec.effective_date == post.post_date
    assert rec.earliest_post_date == post.post_date


def test_an_undated_post_is_dated_from_the_record_its_walk_left_on_disk(
    importer, import_layout,
):
    """#4436: a walk killed before phase 3 wrote the message's record to disk
    but never upserted it, and marked it seen, so no later tick fixed it. The
    repair finds the record under the artist's folder and dates the post — and
    its images — under the post's own source."""
    from backend.app.services.post_record_repair import date_posts_from_records

    import_root, images_root = import_layout
    artist = Artist(name="Alice", slug="alice")
    importer.session.add(artist)
    importer.session.flush()
    importer.session.add(Source(
        artist_id=artist.id, platform="discord",
        url="https://discord.com/channels/1/100",
    ))
    importer.session.flush()
    m = import_root / "Alice" / "20240301_123_01_art.jpg"
    _split(m, "v")
    _sidecar(m, {"category": "discord", "message_id": "123"})
    r = importer.import_one(m)
    assert r.status == "imported"
    post = importer.session.execute(select(Post)).scalar_one()
    assert post.post_date is None and post.source_id is not None

    channel = images_root / "alice" / "discord" / "rewards"
    channel.mkdir(parents=True)
    (channel / "20240301_123_post.json").write_text(json.dumps({
        "category": "discord", "message_id": "123", "message": "",
        "date": "2024-03-01T18:30:00.000000+00:00",
    }))
    (channel / "20240301_999_post.json").write_text("not json")  # skipped, not fatal

    summary = date_posts_from_records(importer.session, importer, images_root)

    assert summary == {"undated": 1, "dated": 1, "no_record": 0}
    importer.session.expire_all()
    post = importer.session.execute(select(Post)).scalar_one()
    rec = importer.session.get(ImageRecord, r.image_id)
    assert post.post_date is not None
    assert rec.earliest_post_date == post.post_date
    # Nothing left undated: the next sweep is an empty query.
    assert date_posts_from_records(importer.session, importer, images_root) == {
        "undated": 0, "dated": 0, "no_record": 0,
    }