CI / lint (push) Successful in 3s
CI / extension-version (push) Successful in 3s
Build images / sign-extension (push) Successful in 4s
Build images / build-agent (push) Successful in 7s
CI / frontend-build (push) Successful in 25s
CI / backend-lint-and-test (push) Successful in 48s
Build images / build-web (push) Successful in 1m14s
Build images / smoke-web (push) Skipped
Build images / build-ml (push) Successful in 2m6s
Build images / promote (push) Skipped
CI / integration (push) Successful in 2m34s
Step 2 of milestone #421. The disk survey counted FOLDERS; this counts ROWS, which is the number that matters — every move in step 3 is a row update, and `ImageRecord.path` is the only pointer at the bytes. `library_layout.py` holds the decision in two shared pieces, and both halves of the consolidation spread them rather than restating them (rule 93, the _x_conditions shape from snippet #3087): - `_misplaced_conditions(root, artist_id, slug)` — rows of one artist whose file is not under that artist's directory. The prefix carries a trailing separator deliberately: without it `ara` matches everything under `arbuzbudesh/`, and one artist reads as fully placed while another's rows are silently skipped. Both are real artists here, hence the test. - `destination_for(path, root, slug)` — where a row's file belongs, or None when it must not be moved: outside the images root, or under one of the reserved stores (`thumbs`, `attachments`, `cookies`, `secrets`, `_backups`, `_quarantine`). Relocating those would move the thumbnail cache or the credential key into an artist folder. `destination_for` diverges from `canonical_subdir` in exactly one case, and the docstring says why: a file at the images ROOT with a known artist moves under that artist here, where the import-time helper leaves it alone. The two answer different questions — an empty subdir at import means no artist was resolved, while a row that already carries an artist_id is an anomaly with a known correct home. The 660 unattributed files have no artist_id at all, so no predicate reaches them; they are counted and left for task #4247. `GET /api/cleanup/layout` exposes it. `?check_disk=1` additionally stats every destination for collisions and missing sources — the conditions the apply refuses on — but it is off by default so the count-only pass answers "how big is this" in seconds instead of timing the request out on NFS. Nothing here writes; a test asserts that against both the row and the file. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
216 lines
7.2 KiB
Python
216 lines
7.2 KiB
Python
"""Milestone #421 — the shared predicate behind the consolidation.
|
|
|
|
`destination_for` is pure and tested without a database; `survey_layout` gets
|
|
the integration treatment because the counts are the number the apply is
|
|
checked against.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import Artist, ImageRecord
|
|
from backend.app.services.library_layout import (
|
|
RESERVED_TOP_LEVEL,
|
|
_misplaced_conditions,
|
|
canonical_dir,
|
|
destination_for,
|
|
survey_layout,
|
|
)
|
|
|
|
ROOT = Path("/images")
|
|
|
|
|
|
# --- destination_for (pure) -------------------------------------------------
|
|
|
|
|
|
def test_destination_rewrites_only_the_artist_segment():
|
|
assert destination_for(
|
|
"/images/Conto/patreon/2026-01_a_Post/x.png", ROOT, "conto"
|
|
) == Path("/images/conto/patreon/2026-01_a_Post/x.png")
|
|
|
|
|
|
def test_destination_is_identity_for_a_row_already_in_place():
|
|
p = "/images/conto/patreon/x.png"
|
|
assert destination_for(p, ROOT, "conto") == Path(p)
|
|
|
|
|
|
def test_destination_pulls_a_root_level_row_under_its_artist():
|
|
"""Diverges from canonical_subdir deliberately: the row CARRIES an
|
|
artist_id, so a file at the root is an anomaly with a known home."""
|
|
assert destination_for("/images/loose.png", ROOT, "conto") == Path(
|
|
"/images/conto/loose.png"
|
|
)
|
|
|
|
|
|
def test_destination_refuses_paths_outside_the_images_root():
|
|
assert destination_for("/srv/elsewhere/x.png", ROOT, "conto") is None
|
|
|
|
|
|
@pytest.mark.parametrize("reserved", sorted(RESERVED_TOP_LEVEL))
|
|
def test_destination_refuses_the_reserved_stores(reserved):
|
|
"""Relocating these would move the thumbnail cache, the attachment blobs
|
|
or the credential key into an artist folder."""
|
|
assert destination_for(f"/images/{reserved}/aa/x.png", ROOT, "conto") is None
|
|
|
|
|
|
def test_destination_is_idempotent():
|
|
once = destination_for("/images/Conto/patreon/x.png", ROOT, "conto")
|
|
assert destination_for(str(once), ROOT, "conto") == once
|
|
|
|
|
|
# --- the predicate ----------------------------------------------------------
|
|
|
|
|
|
def test_canonical_prefix_carries_a_separator():
|
|
"""Without the trailing slash, artist `ara` matches every path under
|
|
`arbuzbudesh/` — one artist reads as fully placed while another's rows
|
|
are silently skipped."""
|
|
conds = _misplaced_conditions(ROOT, 1, "ara")
|
|
rendered = str(conds[-1].compile(compile_kwargs={"literal_binds": True}))
|
|
assert "/images/ara/" in rendered
|
|
|
|
|
|
# --- survey_layout (integration) --------------------------------------------
|
|
#
|
|
# Marked per-test rather than with a module-level `pytestmark`: the
|
|
# destination_for cases above are pure and belong in the fast unit lane.
|
|
|
|
|
|
def _artist(db, name, slug):
|
|
a = Artist(name=name, slug=slug)
|
|
db.add(a)
|
|
db.flush()
|
|
return a
|
|
|
|
|
|
def _image(db, path, artist=None, n=0):
|
|
rec = ImageRecord(
|
|
path=path, sha256=f"{n:064d}", size_bytes=1, mime="image/png",
|
|
width=10, height=10, origin="imported_filesystem",
|
|
integrity_status="unknown",
|
|
artist_id=artist.id if artist else None,
|
|
)
|
|
db.add(rec)
|
|
db.flush()
|
|
return rec
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_splits_canonical_from_misplaced(db_sync):
|
|
conto = _artist(db_sync, "Conto", "conto")
|
|
_image(db_sync, "/images/conto/patreon/a.png", conto, 1)
|
|
_image(db_sync, "/images/Conto/patreon/b.png", conto, 2)
|
|
_image(db_sync, "/images/Conto/patreon/c.png", conto, 3)
|
|
|
|
report = survey_layout(db_sync, ROOT, check_disk=False)
|
|
|
|
assert report.misplaced_rows == 2
|
|
assert report.canonical_rows == 1
|
|
row = next(a for a in report.artists if a.slug == "conto")
|
|
assert row.stray_dirs == ["Conto"]
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_does_not_confuse_a_prefix_sharing_artist(db_sync):
|
|
"""`ara` vs `arbuzbudesh` — the reason the predicate anchors on a
|
|
separator. Both are real artists in the operator's library."""
|
|
ara = _artist(db_sync, "Ara", "ara")
|
|
arbuz = _artist(db_sync, "ArbuzBudesh", "arbuzbudesh")
|
|
_image(db_sync, "/images/ara/x.png", ara, 4)
|
|
_image(db_sync, "/images/arbuzbudesh/y.png", arbuz, 5)
|
|
|
|
report = survey_layout(db_sync, ROOT, check_disk=False)
|
|
|
|
assert report.misplaced_rows == 0
|
|
assert report.canonical_rows == 2
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_counts_two_rows_landing_on_one_destination(db_sync):
|
|
"""A collision is the case the apply must refuse, so the report has to
|
|
surface it rather than promise a move that cannot happen."""
|
|
sticky = _artist(db_sync, "StickySpoodge", "stickyspoodge")
|
|
_image(db_sync, "/images/StickySpoodge/p/dup.png", sticky, 6)
|
|
_image(db_sync, "/images/Stickyspoodge/p/dup.png", sticky, 7)
|
|
|
|
report = survey_layout(db_sync, ROOT, check_disk=False)
|
|
|
|
assert report.collision_count == 1
|
|
row = next(a for a in report.artists if a.slug == "stickyspoodge")
|
|
assert row.collisions == ["/images/stickyspoodge/p/dup.png"]
|
|
assert row.stray_dirs == ["StickySpoodge", "Stickyspoodge"]
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_reports_unattributed_rows_without_moving_them(db_sync):
|
|
"""The 660 loose root files have no artist_id, so no predicate reaches
|
|
them. They are counted, and left for task #4247."""
|
|
_image(db_sync, "/images/orphan.png", None, 8)
|
|
|
|
report = survey_layout(db_sync, ROOT, check_disk=False)
|
|
|
|
assert report.unattributed_rows == 1
|
|
assert report.misplaced_rows == 0
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_refuses_a_row_under_a_reserved_store(db_sync):
|
|
thumbs = _artist(db_sync, "Thumbsy", "thumbsy")
|
|
_image(db_sync, "/images/thumbs/aa/weird.png", thumbs, 9)
|
|
|
|
report = survey_layout(db_sync, ROOT, check_disk=False)
|
|
|
|
assert report.unmovable == 1
|
|
row = next(a for a in report.artists if a.slug == "thumbsy")
|
|
assert row.misplaced_rows == 1
|
|
assert row.collisions == []
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_counts_a_missing_source_file(db_sync, tmp_path):
|
|
"""check_disk is what separates "would move" from "can move"."""
|
|
gone = _artist(db_sync, "Gone", "gone")
|
|
_image(db_sync, str(tmp_path / "Gone" / "missing.png"), gone, 10)
|
|
|
|
report = survey_layout(db_sync, tmp_path, check_disk=True)
|
|
|
|
assert report.missing_files == 1
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_flags_a_destination_that_already_exists(db_sync, tmp_path):
|
|
occupied = _artist(db_sync, "Occupied", "occupied")
|
|
src = tmp_path / "Occupied" / "x.png"
|
|
src.parent.mkdir(parents=True)
|
|
src.write_bytes(b"src")
|
|
dest = canonical_dir(tmp_path, "occupied") / "x.png"
|
|
dest.parent.mkdir(parents=True)
|
|
dest.write_bytes(b"already here")
|
|
_image(db_sync, str(src), occupied, 11)
|
|
|
|
report = survey_layout(db_sync, tmp_path, check_disk=True)
|
|
|
|
assert report.collision_count == 1
|
|
assert dest.read_bytes() == b"already here" # read-only: nothing moved
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_survey_is_read_only(db_sync, tmp_path):
|
|
a = _artist(db_sync, "Reader", "reader")
|
|
src = tmp_path / "Reader" / "x.png"
|
|
src.parent.mkdir(parents=True)
|
|
src.write_bytes(b"x")
|
|
rec = _image(db_sync, str(src), a, 12)
|
|
before = rec.path
|
|
|
|
survey_layout(db_sync, tmp_path, check_disk=True)
|
|
|
|
db_sync.expire_all()
|
|
assert db_sync.get(ImageRecord, rec.id).path == before
|
|
assert src.exists()
|
|
assert db_sync.execute(
|
|
select(ImageRecord.path).where(ImageRecord.id == rec.id)
|
|
).scalar_one() == before
|