revert: remove the placement reconciler — it manufactured the problem it solved

Milestone #421 built a sweep that compared each image's `artist_id` to the name of the directory holding its file, and called every mismatch a misplaced image. It reported 33,789 of 63,605 as wrongly filed. That number described the comparison, not the library. What it actually was: 32,475 (97.1%) one artist's own folder, spelled differently — Telepurte/ vs telepurte/. Same artist, same art. 657 ( 2.0%) loose at the images root 328 ( 1.0%) in a folder named after a different artist And the 1% did not mean what the tool assumed either. `ImageProvenance` records the post and source every file was downloaded from — the authoritative answer, which the tool never consulted. Querying it for all 328: 144 provenance agrees with the record (move would be right) 87 provenance agrees with the FOLDER (the record is wrong; move wrong) 53 provenance names SEVERAL artists (no single correct folder) 41 no provenance at all 3 agrees with neither So the sweep would have misfiled or arbitrarily picked for ~41% of the only set it was really needed for. The system already knew where each file came from; the tool inferred it from a column and a directory name instead. Operator, 2026-09-21: *"the current system consistently records where items are and where they came from this is just complicating something works and doesn't need fixing."* Correct on both counts. Removed: the service, the tasks, the model and migration 0099's table, the /api/cleanup/layout and /placement/* endpoints, the Maintenance card and its store actions, and the tests. 0100 drops the table (rule #22 — no legacy). KEPT deliberately, per the operator: - `utils.paths.canonical_subdir` — new filesystem imports derive their directory from the artist's slug, matching what the downloader always did. Not part of this tool; removing it would be churn that fixes nothing. - The 327 files run 1 moved (InsoUwu/ -> insouwu/). Same artist either way, and the gallery renders them correctly. - Everything from #4223 (three-gate dedup, 256-bit pHash) and #4234 (backup credential exclusion). Those fixed problems that were actually reported. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-21 18:17:13 -04:00
parent 2dd9b956d5
commit 11a01a9686
14 changed files with 111 additions and 1850 deletions
@@ -1,167 +0,0 @@
-"""The placement reconciler's task + API surface (milestone #421, slice 3b).
-
-The move logic itself is covered in tests/test_library_layout.py; this module
-covers the wrapper — that the tasks are registered and routed, that the
-endpoints gate on run state, and that a list response stays small.
-"""
-
-import pytest
-from sqlalchemy import select
-
-import backend.app.tasks.library_placement  # noqa: F401 — register tasks
-from backend.app.celery_app import celery
-from backend.app.models import Artist, LibraryPlacementRun
-
-pytestmark = pytest.mark.integration
-
-_TASKS = (
-    "backend.app.tasks.library_placement.plan_placement",
-    "backend.app.tasks.library_placement.apply_placement",
-    "backend.app.tasks.library_placement.revert_placement",
-)
-
-
-@pytest.mark.parametrize("name", _TASKS)
-def test_placement_tasks_are_registered(name):
-    assert name in celery.tasks
-
-
-def test_placement_runs_on_the_long_maintenance_lane():
-    """33k renames must not sit in the quick lane, which is where the
-    self-healing sweeps live (the 2026-06-07 starvation)."""
-    routes = celery.conf.task_routes
-    assert routes["backend.app.tasks.library_placement.*"] == {
-        "queue": "maintenance_long"
-    }
-
-
-def _run(db, status="ready", moves=None, artist_id=None):
-    """Adds the row; the caller awaits the COMMIT.
-
-    Commit, not flush: the app under test runs on its own session and
-    connection, so a flush that stays inside this test's transaction is
-    invisible to the endpoint — the row simply is not there yet. Same reason
-    `_seed_runs` in test_api_system_backup commits."""
-    run = LibraryPlacementRun(
-        status=status, artist_id=artist_id, moves=moves or [],
-        planned_count=len(moves or []),
-    )
-    db.add(run)
-    return run
-
-
-@pytest.mark.asyncio
-async def test_runs_list_omits_the_moves(client, db):
-    """An applied whole-library run carries tens of thousands of entries.
-    Fine in Postgres, wrong in every list response."""
-    run = LibraryPlacementRun(
-        status="applied",
-        moves=[{"image_id": 1, "from": "/images/A/x.png", "to": "/images/a/x.png"}],
-        planned_count=1, moved_count=1,
-    )
-    db.add(run)
-    await db.commit()
-
-    resp = await client.get("/api/cleanup/placement/runs")
-    assert resp.status_code == 200
-    body = await resp.get_json()
-    assert body["runs"][0]["planned_count"] == 1
-    assert "moves" not in body["runs"][0]
-
-
-@pytest.mark.asyncio
-async def test_run_detail_carries_the_moves(client, db):
-    """The detail IS the preview the operator reads before agreeing."""
-    run = LibraryPlacementRun(
-        status="ready",
-        moves=[{"image_id": 7, "from": "/images/Conto/x.png", "to": "/images/conto/x.png"}],
-        planned_count=1,
-    )
-    db.add(run)
-    await db.commit()
-
-    resp = await client.get(f"/api/cleanup/placement/runs/{run.id}")
-    assert resp.status_code == 200
-    body = await resp.get_json()
-    assert body["moves"][0]["from"] == "/images/Conto/x.png"
-    assert body["moves"][0]["to"] == "/images/conto/x.png"
-
-
-@pytest.mark.asyncio
-async def test_run_detail_404s_for_an_unknown_run(client):
-    resp = await client.get("/api/cleanup/placement/runs/999999")
-    assert resp.status_code == 404
-
-
-@pytest.mark.asyncio
-async def test_plan_rejects_a_non_integer_artist(client):
-    resp = await client.post(
-        "/api/cleanup/placement/plan", json={"artist_id": "conto"},
-    )
-    assert resp.status_code == 400
-    assert (await resp.get_json())["error"] == "invalid_artist_id"
-
-
-@pytest.mark.asyncio
-async def test_plan_accepts_an_artist_scope(client, db, monkeypatch):
-    sent = {}
-    from backend.app.tasks import library_placement
-
-    monkeypatch.setattr(
-        library_placement.plan_placement, "delay",
-        lambda artist_id=None: sent.update(artist_id=artist_id),
-    )
-    artist = Artist(name="Conto", slug="conto")
-    db.add(artist)
-    await db.commit()
-
-    resp = await client.post(
-        "/api/cleanup/placement/plan", json={"artist_id": artist.id},
-    )
-    assert resp.status_code == 202
-    assert sent["artist_id"] == artist.id
-
-
-@pytest.mark.asyncio
-async def test_apply_refuses_a_run_that_is_not_ready(client, db):
-    """The gate is here as well as in the service — an applied run must not
-    be re-applied by a stray POST."""
-    run = _run(db, status="applied")
-    await db.commit()
-
-    resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/apply")
-    assert resp.status_code == 400
-    assert (await resp.get_json())["error"] == "not_ready"
-
-
-@pytest.mark.asyncio
-async def test_revert_refuses_a_run_that_was_never_applied(client, db):
-    run = _run(db, status="ready")
-    await db.commit()
-
-    resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/revert")
-    assert resp.status_code == 400
-    assert (await resp.get_json())["error"] == "not_applied"
-
-
-@pytest.mark.asyncio
-async def test_apply_dispatches_for_a_ready_run(client, db, monkeypatch):
-    sent = {}
-    from backend.app.tasks import library_placement
-
-    monkeypatch.setattr(
-        library_placement.apply_placement, "delay",
-        lambda run_id: sent.update(run_id=run_id),
-    )
-    run = _run(db, status="ready")
-    await db.commit()
-
-    resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/apply")
-    assert resp.status_code == 202
-    assert sent["run_id"] == run.id
-    # Dispatch only — the endpoint must not have moved anything itself.
-    still = (await db.execute(
-        select(LibraryPlacementRun.status)
-        .where(LibraryPlacementRun.id == run.id)
-    )).scalar_one()
-    assert still == "ready"
@@ -1,403 +0,0 @@
-"""Milestone #421 — the shared predicate behind the consolidation.
-
-`destination_for` is pure and tested without a database; `survey_layout` gets
-the integration treatment because the counts are the number the apply is
-checked against.
-"""
-
-from pathlib import Path
-
-import pytest
-from sqlalchemy import select
-
-from backend.app.models import Artist, ImageRecord
-from backend.app.services.library_layout import (
-    RESERVED_TOP_LEVEL,
-    _misplaced_conditions,
-    canonical_dir,
-    destination_for,
-    survey_layout,
-)
-
-ROOT = Path("/images")
-
-
-# --- destination_for (pure) -------------------------------------------------
-
-
-def test_destination_rewrites_only_the_artist_segment():
-    assert destination_for(
-        "/images/Conto/patreon/2026-01_a_Post/x.png", ROOT, "conto"
-    ) == Path("/images/conto/patreon/2026-01_a_Post/x.png")
-
-
-def test_destination_is_identity_for_a_row_already_in_place():
-    p = "/images/conto/patreon/x.png"
-    assert destination_for(p, ROOT, "conto") == Path(p)
-
-
-def test_destination_pulls_a_root_level_row_under_its_artist():
-    """Diverges from canonical_subdir deliberately: the row CARRIES an
-    artist_id, so a file at the root is an anomaly with a known home."""
-    assert destination_for("/images/loose.png", ROOT, "conto") == Path(
-        "/images/conto/loose.png"
-    )
-
-
-def test_destination_refuses_paths_outside_the_images_root():
-    assert destination_for("/srv/elsewhere/x.png", ROOT, "conto") is None
-
-
-@pytest.mark.parametrize("reserved", sorted(RESERVED_TOP_LEVEL))
-def test_destination_refuses_the_reserved_stores(reserved):
-    """Relocating these would move the thumbnail cache, the attachment blobs
-    or the credential key into an artist folder."""
-    assert destination_for(f"/images/{reserved}/aa/x.png", ROOT, "conto") is None
-
-
-def test_destination_is_idempotent():
-    once = destination_for("/images/Conto/patreon/x.png", ROOT, "conto")
-    assert destination_for(str(once), ROOT, "conto") == once
-
-
-# --- the predicate ----------------------------------------------------------
-
-
-def test_canonical_prefix_carries_a_separator():
-    """Without the trailing slash, artist `ara` matches every path under
-    `arbuzbudesh/` — one artist reads as fully placed while another's rows
-    are silently skipped."""
-    conds = _misplaced_conditions(ROOT, 1, "ara")
-    rendered = str(conds[-1].compile(compile_kwargs={"literal_binds": True}))
-    assert "/images/ara/" in rendered
-
-
-# --- survey_layout (integration) --------------------------------------------
-#
-# Marked per-test rather than with a module-level `pytestmark`: the
-# destination_for cases above are pure and belong in the fast unit lane.
-
-
-def _artist(db, name, slug):
-    a = Artist(name=name, slug=slug)
-    db.add(a)
-    db.flush()
-    return a
-
-
-def _image(db, path, artist=None, n=0):
-    rec = ImageRecord(
-        path=path, sha256=f"{n:064d}", size_bytes=1, mime="image/png",
-        width=10, height=10, origin="imported_filesystem",
-        integrity_status="unknown",
-        artist_id=artist.id if artist else None,
-    )
-    db.add(rec)
-    db.flush()
-    return rec
-
-
-@pytest.mark.integration
-def test_survey_splits_canonical_from_misplaced(db_sync):
-    conto = _artist(db_sync, "Conto", "conto")
-    _image(db_sync, "/images/conto/patreon/a.png", conto, 1)
-    _image(db_sync, "/images/Conto/patreon/b.png", conto, 2)
-    _image(db_sync, "/images/Conto/patreon/c.png", conto, 3)
-
-    report = survey_layout(db_sync, ROOT, check_disk=False)
-
-    assert report.misplaced_rows == 2
-    assert report.canonical_rows == 1
-    row = next(a for a in report.artists if a.slug == "conto")
-    assert row.stray_dirs == ["Conto"]
-
-
-@pytest.mark.integration
-def test_survey_does_not_confuse_a_prefix_sharing_artist(db_sync):
-    """`ara` vs `arbuzbudesh` — the reason the predicate anchors on a
-    separator. Both are real artists in the operator's library."""
-    ara = _artist(db_sync, "Ara", "ara")
-    arbuz = _artist(db_sync, "ArbuzBudesh", "arbuzbudesh")
-    _image(db_sync, "/images/ara/x.png", ara, 4)
-    _image(db_sync, "/images/arbuzbudesh/y.png", arbuz, 5)
-
-    report = survey_layout(db_sync, ROOT, check_disk=False)
-
-    assert report.misplaced_rows == 0
-    assert report.canonical_rows == 2
-
-
-@pytest.mark.integration
-def test_survey_counts_two_rows_landing_on_one_destination(db_sync):
-    """A collision is the case the apply must refuse, so the report has to
-    surface it rather than promise a move that cannot happen."""
-    sticky = _artist(db_sync, "StickySpoodge", "stickyspoodge")
-    _image(db_sync, "/images/StickySpoodge/p/dup.png", sticky, 6)
-    _image(db_sync, "/images/Stickyspoodge/p/dup.png", sticky, 7)
-
-    report = survey_layout(db_sync, ROOT, check_disk=False)
-
-    assert report.collision_count == 1
-    row = next(a for a in report.artists if a.slug == "stickyspoodge")
-    assert row.collisions == ["/images/stickyspoodge/p/dup.png"]
-    assert row.stray_dirs == ["StickySpoodge", "Stickyspoodge"]
-
-
-@pytest.mark.integration
-def test_survey_reports_unattributed_rows_without_moving_them(db_sync):
-    """The 660 loose root files have no artist_id, so no predicate reaches
-    them. They are counted, and left for task #4247."""
-    _image(db_sync, "/images/orphan.png", None, 8)
-
-    report = survey_layout(db_sync, ROOT, check_disk=False)
-
-    assert report.unattributed_rows == 1
-    assert report.misplaced_rows == 0
-
-
-@pytest.mark.integration
-def test_survey_refuses_a_row_under_a_reserved_store(db_sync):
-    thumbs = _artist(db_sync, "Thumbsy", "thumbsy")
-    _image(db_sync, "/images/thumbs/aa/weird.png", thumbs, 9)
-
-    report = survey_layout(db_sync, ROOT, check_disk=False)
-
-    assert report.unmovable == 1
-    row = next(a for a in report.artists if a.slug == "thumbsy")
-    assert row.misplaced_rows == 1
-    assert row.collisions == []
-
-
-@pytest.mark.integration
-def test_survey_counts_a_missing_source_file(db_sync, tmp_path):
-    """check_disk is what separates "would move" from "can move"."""
-    gone = _artist(db_sync, "Gone", "gone")
-    _image(db_sync, str(tmp_path / "Gone" / "missing.png"), gone, 10)
-
-    report = survey_layout(db_sync, tmp_path, check_disk=True)
-
-    assert report.missing_files == 1
-
-
-@pytest.mark.integration
-def test_survey_flags_a_destination_that_already_exists(db_sync, tmp_path):
-    occupied = _artist(db_sync, "Occupied", "occupied")
-    src = tmp_path / "Occupied" / "x.png"
-    src.parent.mkdir(parents=True)
-    src.write_bytes(b"src")
-    dest = canonical_dir(tmp_path, "occupied") / "x.png"
-    dest.parent.mkdir(parents=True)
-    dest.write_bytes(b"already here")
-    _image(db_sync, str(src), occupied, 11)
-
-    report = survey_layout(db_sync, tmp_path, check_disk=True)
-
-    assert report.collision_count == 1
-    assert dest.read_bytes() == b"already here"  # read-only: nothing moved
-
-
-@pytest.mark.integration
-def test_survey_is_read_only(db_sync, tmp_path):
-    a = _artist(db_sync, "Reader", "reader")
-    src = tmp_path / "Reader" / "x.png"
-    src.parent.mkdir(parents=True)
-    src.write_bytes(b"x")
-    rec = _image(db_sync, str(src), a, 12)
-    before = rec.path
-
-    survey_layout(db_sync, tmp_path, check_disk=True)
-
-    db_sync.expire_all()
-    assert db_sync.get(ImageRecord, rec.id).path == before
-    assert src.exists()
-    assert db_sync.execute(
-        select(ImageRecord.path).where(ImageRecord.id == rec.id)
-    ).scalar_one() == before
-
-
-# --- plan / apply / revert (#4246) ------------------------------------------
-
-
-def _staged(db, tmp_path, slug, stray, name="x.png", n=100):
-    """An artist with one file sitting in `stray`'s directory."""
-    artist = _artist(db, slug.title(), slug)
-    src = tmp_path / stray / name
-    src.parent.mkdir(parents=True, exist_ok=True)
-    src.write_bytes(b"pixels")
-    rec = _image(db, str(src), artist, n)
-    return artist, rec, src
-
-
-@pytest.mark.integration
-def test_plan_records_where_each_file_came_from(db_sync, tmp_path):
-    from backend.app.services.library_layout import plan_placement
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=20)
-    run = plan_placement(db_sync, tmp_path)
-
-    assert run.status == "ready"
-    assert run.planned_count == 1
-    assert run.moves == [{
-        "image_id": rec.id,
-        "from": str(src),
-        "to": str(tmp_path / "conto" / "x.png"),
-    }]
-    # Planning touches nothing.
-    assert src.exists()
-    assert db_sync.get(ImageRecord, rec.id).path == str(src)
-
-
-@pytest.mark.integration
-def test_apply_moves_file_and_row_together(db_sync, tmp_path):
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=21)
-    run = apply_run(db_sync, plan_placement(db_sync, tmp_path))
-
-    dest = tmp_path / "conto" / "x.png"
-    assert run.status == "applied"
-    assert run.moved_count == 1 and run.refused_count == 0
-    assert dest.exists() and not src.exists()
-    db_sync.expire_all()
-    assert db_sync.get(ImageRecord, rec.id).path == str(dest)
-
-
-@pytest.mark.integration
-def test_revert_puts_it_back(db_sync, tmp_path):
-    """The whole reason `from` is retained: do one artist, look, undo."""
-    from backend.app.services.library_layout import (
-        apply_run,
-        plan_placement,
-        revert_run,
-    )
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=22)
-    run = revert_run(db_sync, apply_run(db_sync, plan_placement(db_sync, tmp_path)))
-
-    assert run.status == "reverted"
-    assert src.exists()
-    assert not (tmp_path / "conto" / "x.png").exists()
-    db_sync.expire_all()
-    assert db_sync.get(ImageRecord, rec.id).path == str(src)
-
-
-@pytest.mark.integration
-def test_apply_refuses_a_row_that_moved_since_planning(db_sync, tmp_path):
-    """A supersede or an earlier run can rewrite a path between plan and
-    apply. The stale entry is declined, not forced."""
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=23)
-    run = plan_placement(db_sync, tmp_path)
-
-    elsewhere = tmp_path / "conto" / "already-here.png"
-    elsewhere.parent.mkdir(parents=True, exist_ok=True)
-    elsewhere.write_bytes(b"pixels")
-    rec.path = str(elsewhere)
-    db_sync.flush()
-
-    run = apply_run(db_sync, run)
-
-    assert run.moved_count == 0 and run.refused_count == 1
-    assert run.refusals[0]["reason"] == "row moved since planning"
-    assert src.exists()  # untouched
-
-
-@pytest.mark.integration
-def test_apply_never_overwrites_an_occupied_destination(db_sync, tmp_path):
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=24)
-    run = plan_placement(db_sync, tmp_path)
-
-    squatter = tmp_path / "conto" / "x.png"
-    squatter.parent.mkdir(parents=True, exist_ok=True)
-    squatter.write_bytes(b"someone else")
-
-    run = apply_run(db_sync, run)
-
-    assert run.refused_count == 1
-    assert run.refusals[0]["reason"] == "destination occupied"
-    assert squatter.read_bytes() == b"someone else"
-    db_sync.expire_all()
-    assert db_sync.get(ImageRecord, rec.id).path == str(src)
-
-
-@pytest.mark.integration
-def test_apply_leaves_the_row_alone_when_the_source_is_gone(db_sync, tmp_path):
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    _, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=25)
-    run = plan_placement(db_sync, tmp_path)
-    src.unlink()
-
-    run = apply_run(db_sync, run)
-
-    assert run.refusals[0]["reason"] == "source missing"
-    db_sync.expire_all()
-    # The row still points at the missing file rather than at a file that
-    # was never created — a broken row is recoverable, a lying one is not.
-    assert db_sync.get(ImageRecord, rec.id).path == str(src)
-
-
-@pytest.mark.integration
-def test_plan_scopes_to_one_artist(db_sync, tmp_path):
-    """Per-artist scope is what makes this incremental instead of one
-    irreversible sweep."""
-    from backend.app.services.library_layout import plan_placement
-
-    conto, _, _ = _staged(db_sync, tmp_path, "conto", "Conto", n=26)
-    _staged(db_sync, tmp_path, "maewix", "Maewix", name="y.png", n=27)
-
-    run = plan_placement(db_sync, tmp_path, artist_id=conto.id)
-
-    assert run.planned_count == 1
-    assert run.artist_id == conto.id
-    assert "Conto" in run.moves[0]["from"]
-
-
-@pytest.mark.integration
-def test_plan_skips_both_rows_when_two_want_one_destination(db_sync, tmp_path):
-    """Which of two colliding rows 'wins' is not this sweep's call."""
-    from backend.app.services.library_layout import plan_placement
-
-    artist = _artist(db_sync, "Sticky", "sticky")
-    for stray, n in (("StickySpoodge", 28), ("Stickyspoodge", 29)):
-        p = tmp_path / stray / "dup.png"
-        p.parent.mkdir(parents=True, exist_ok=True)
-        p.write_bytes(b"pixels")
-        _image(db_sync, str(p), artist, n)
-
-    run = plan_placement(db_sync, tmp_path)
-
-    assert run.planned_count == 0
-
-
-@pytest.mark.integration
-def test_thumbnails_do_not_move(db_sync, tmp_path):
-    """Thumbs are sha-addressed (`thumbs/<xx>/<sha>.jpg`), not path-keyed, so
-    a placement move must not touch them. Pinned so nobody 'fixes' it."""
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    artist, rec, _ = _staged(db_sync, tmp_path, "conto", "Conto", n=30)
-    thumb = tmp_path / "thumbs" / "ab" / "abc.jpg"
-    thumb.parent.mkdir(parents=True, exist_ok=True)
-    thumb.write_bytes(b"thumb")
-    rec.thumbnail_path = str(thumb)
-    db_sync.flush()
-
-    apply_run(db_sync, plan_placement(db_sync, tmp_path))
-
-    db_sync.expire_all()
-    assert thumb.exists()
-    assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb)
-
-
-@pytest.mark.integration
-def test_apply_refuses_a_run_that_is_not_ready(db_sync, tmp_path):
-    from backend.app.services.library_layout import apply_run, plan_placement
-
-    _staged(db_sync, tmp_path, "conto", "Conto", n=31)
-    run = apply_run(db_sync, plan_placement(db_sync, tmp_path))
-    with pytest.raises(ValueError):
-        apply_run(db_sync, run)