Placement reconciler: put every image in its own artist's folder, one artist at a time #256
@@ -0,0 +1,99 @@
|
|||||||
|
"""library_placement_run — the placement reconciler's plan/apply/undo ledger.
|
||||||
|
|
||||||
|
Milestone #421 step 3. The survey (#4245) measured 33,789 ImageRecord rows
|
||||||
|
sitting outside their artist's canonical directory, across 56 artists. This
|
||||||
|
table holds one run of the sweep that trues them up: the plan, what it did,
|
||||||
|
and where every file came from.
|
||||||
|
|
||||||
|
## Why the moves live in a table rather than a log line
|
||||||
|
|
||||||
|
`ImageRecord.path` is the only pointer at the bytes, so a move rewrites the
|
||||||
|
row. Once that write lands, the previous location exists nowhere — unless it
|
||||||
|
was recorded first. `moves` is that record, which is what makes a 33,789-file
|
||||||
|
operation something the operator can undo per artist after looking at the
|
||||||
|
result, rather than a one-way door.
|
||||||
|
|
||||||
|
An `applied` row is therefore HISTORY, not state (lesson #4226). Any future
|
||||||
|
retention on this table may prune `ready`, `cancelled` and `error` runs; an
|
||||||
|
`applied` one is only disposable once someone decides undo is no longer
|
||||||
|
wanted. That is deliberately not a timer's decision, and no pruning is added
|
||||||
|
here.
|
||||||
|
|
||||||
|
Revision ID: 0099
|
||||||
|
Revises: 0098
|
||||||
|
Create Date: 2026-09-21
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0099"
|
||||||
|
down_revision: Union[str, None] = "0098"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"library_placement_run",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), server_default="running",
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
# SET NULL, not CASCADE: deleting an artist must not destroy the
|
||||||
|
# record of where their files were moved.
|
||||||
|
sa.Column("artist_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"planned_count", sa.Integer(), server_default="0", nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"moved_count", sa.Integer(), server_default="0", nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"refused_count", sa.Integer(), server_default="0", nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"moves", postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"refusals", postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["artist_id"], ["artist.id"],
|
||||||
|
name="fk_library_placement_run_artist_id", ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_placement_run_status", "library_placement_run", ["status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_placement_run_artist_id", "library_placement_run",
|
||||||
|
["artist_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Dropping this table destroys the only record of where moved files came
|
||||||
|
# from. That is correct for a downgrade — the code that reads it is going
|
||||||
|
# away too — but it is worth saying out loud rather than discovering.
|
||||||
|
op.drop_index(
|
||||||
|
"ix_library_placement_run_artist_id",
|
||||||
|
table_name="library_placement_run",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_library_placement_run_status", table_name="library_placement_run",
|
||||||
|
)
|
||||||
|
op.drop_table("library_placement_run")
|
||||||
@@ -29,7 +29,7 @@ from quart import Blueprint, jsonify, request
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import LibraryAuditRun
|
from ..models import LibraryAuditRun, LibraryPlacementRun
|
||||||
from ..services import cleanup_service, library_layout
|
from ..services import cleanup_service, library_layout
|
||||||
from ._responses import error_response as _bad
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
@@ -218,3 +218,91 @@ async def layout_survey():
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
return jsonify({**report.as_dict(), "checked_disk": check_disk})
|
return jsonify({**report.as_dict(), "checked_disk": check_disk})
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_placement_run(run: LibraryPlacementRun, *, moves: bool = False) -> dict:
|
||||||
|
"""`moves` is opt-in: an applied whole-library run carries tens of
|
||||||
|
thousands of entries, which is a fine thing to hold in Postgres and a
|
||||||
|
poor thing to put in every list response."""
|
||||||
|
out = {
|
||||||
|
"id": run.id,
|
||||||
|
"status": run.status,
|
||||||
|
"artist_id": run.artist_id,
|
||||||
|
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||||
|
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||||
|
"planned_count": run.planned_count,
|
||||||
|
"moved_count": run.moved_count,
|
||||||
|
"refused_count": run.refused_count,
|
||||||
|
"refusals": run.refusals or [],
|
||||||
|
"error": run.error,
|
||||||
|
}
|
||||||
|
if moves:
|
||||||
|
out["moves"] = run.moves or []
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/placement/runs", methods=["GET"])
|
||||||
|
async def placement_runs():
|
||||||
|
"""Newest first. Without `moves`, so the list stays small."""
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "25")), 100)
|
||||||
|
except ValueError:
|
||||||
|
return _bad("invalid_limit")
|
||||||
|
async with get_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(LibraryPlacementRun)
|
||||||
|
.order_by(LibraryPlacementRun.id.desc()).limit(limit)
|
||||||
|
)).scalars().all()
|
||||||
|
return jsonify({"runs": [_serialize_placement_run(r) for r in rows]})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/placement/runs/<int:run_id>", methods=["GET"])
|
||||||
|
async def placement_run(run_id: int):
|
||||||
|
"""One run WITH its moves — this is the preview the operator reads before
|
||||||
|
agreeing, and the record of what happened afterwards."""
|
||||||
|
async with get_session() as session:
|
||||||
|
run = await session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
return jsonify(_serialize_placement_run(run, moves=True))
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/placement/plan", methods=["POST"])
|
||||||
|
async def placement_plan():
|
||||||
|
"""Queue a planning run. `artist_id` scopes it to one artist, which is the
|
||||||
|
intended use: do one, look at the gallery, then continue or revert."""
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
artist_id = body.get("artist_id")
|
||||||
|
if artist_id is not None and not isinstance(artist_id, int):
|
||||||
|
return _bad("invalid_artist_id")
|
||||||
|
from ..tasks.library_placement import plan_placement
|
||||||
|
plan_placement.delay(artist_id)
|
||||||
|
return jsonify({"status": "dispatched"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/placement/runs/<int:run_id>/apply", methods=["POST"])
|
||||||
|
async def placement_apply(run_id: int):
|
||||||
|
"""Execute a ready run. This renames files and rewrites rows."""
|
||||||
|
async with get_session() as session:
|
||||||
|
run = await session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
if run.status != "ready":
|
||||||
|
return _bad("not_ready", detail=f"run is {run.status}")
|
||||||
|
from ..tasks.library_placement import apply_placement
|
||||||
|
apply_placement.delay(run_id)
|
||||||
|
return jsonify({"status": "dispatched"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/placement/runs/<int:run_id>/revert", methods=["POST"])
|
||||||
|
async def placement_revert(run_id: int):
|
||||||
|
"""Put an applied run's files back. The reason the ledger is kept."""
|
||||||
|
async with get_session() as session:
|
||||||
|
run = await session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
if run.status != "applied":
|
||||||
|
return _bad("not_applied", detail=f"run is {run.status}")
|
||||||
|
from ..tasks.library_placement import revert_placement
|
||||||
|
revert_placement.delay(run_id)
|
||||||
|
return jsonify({"status": "dispatched"}), 202
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.backup",
|
"backend.app.tasks.backup",
|
||||||
"backend.app.tasks.admin",
|
"backend.app.tasks.admin",
|
||||||
"backend.app.tasks.library_audit",
|
"backend.app.tasks.library_audit",
|
||||||
|
"backend.app.tasks.library_placement",
|
||||||
"backend.app.tasks.translation",
|
"backend.app.tasks.translation",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -62,6 +63,8 @@ def make_celery() -> Celery:
|
|||||||
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||||
|
# 33k renames on NFS: long lane, same as backups.
|
||||||
|
"backend.app.tasks.library_placement.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||||
# Translation backfill hits the LLM (~1–6s/item) → the long lane so it
|
# Translation backfill hits the LLM (~1–6s/item) → the long lane so it
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from .import_batch import ImportBatch
|
|||||||
from .import_settings import ImportSettings
|
from .import_settings import ImportSettings
|
||||||
from .import_task import ImportTask
|
from .import_task import ImportTask
|
||||||
from .library_audit_run import LibraryAuditRun
|
from .library_audit_run import LibraryAuditRun
|
||||||
|
from .library_placement_run import LibraryPlacementRun
|
||||||
from .membership_sync import MembershipSync
|
from .membership_sync import MembershipSync
|
||||||
from .ml_settings import MLSettings
|
from .ml_settings import MLSettings
|
||||||
from .patreon_failed_media import PatreonFailedMedia
|
from .patreon_failed_media import PatreonFailedMedia
|
||||||
@@ -85,6 +86,7 @@ __all__ = [
|
|||||||
"ImportTask",
|
"ImportTask",
|
||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
"LibraryAuditRun",
|
"LibraryAuditRun",
|
||||||
|
"LibraryPlacementRun",
|
||||||
"MembershipSync",
|
"MembershipSync",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
"HeadAutoApplyRun",
|
"HeadAutoApplyRun",
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
"""LibraryPlacementRun — one run of the placement reconciler (milestone #421).
|
||||||
|
|
||||||
|
The library is keyed on the Artist row's `slug`, one directory per artist.
|
||||||
|
Every writer agrees on that now (`utils.paths.canonical_subdir`, task #4244),
|
||||||
|
but ~33,789 rows were written under older rules and sit in some other
|
||||||
|
artist's directory. This row is a run of the sweep that trues them up.
|
||||||
|
|
||||||
|
State machine, mirroring LibraryAuditRun:
|
||||||
|
|
||||||
|
running -> ready -> applied -> reverted
|
||||||
|
\\-> cancelled
|
||||||
|
(any) -> error
|
||||||
|
|
||||||
|
## The `moves` column does three jobs
|
||||||
|
|
||||||
|
`moves` is the plan: `[{"image_id": 1, "from": "...", "to": "..."}, ...]`.
|
||||||
|
|
||||||
|
1. **Preview.** It is what the operator reads before agreeing.
|
||||||
|
2. **Apply.** The apply executes THIS list rather than re-deriving the set,
|
||||||
|
so the preview cannot describe a different set from the apply. That is
|
||||||
|
rule 93's guarantee reached the way LibraryAuditRun reaches it — the
|
||||||
|
plan is materialised, not recomputed.
|
||||||
|
3. **Revert.** `from` is retained, so a batch that looks wrong in the
|
||||||
|
gallery goes back where it came from.
|
||||||
|
|
||||||
|
## An applied run IS the undo ledger — it must never be pruned
|
||||||
|
|
||||||
|
This is the trap lesson #4226 names: a record that answers both "what is the
|
||||||
|
current plan" and "what happened" gets deleted by whatever forgets the first.
|
||||||
|
A `ready` run is disposable state. An `applied` run is HISTORY, and it is the
|
||||||
|
only record of where 33,789 files used to be — delete it and the moves become
|
||||||
|
irreversible.
|
||||||
|
|
||||||
|
No pruning exists for this table today, and that is deliberate. If retention
|
||||||
|
is ever added here, it may prune `ready`, `cancelled` and `error` runs; an
|
||||||
|
`applied` run is only safe to drop once someone decides the moves are settled
|
||||||
|
and undo is no longer wanted, which is an operator decision and not a
|
||||||
|
timer's.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, func, text
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryPlacementRun(Base):
|
||||||
|
__tablename__ = "library_placement_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
server_default="running",
|
||||||
|
)
|
||||||
|
# running | ready | applied | reverted | cancelled | error
|
||||||
|
|
||||||
|
# Scope. NULL = the whole library; set = one artist, which is how this is
|
||||||
|
# meant to be used — do one artist, look at it in the gallery, continue or
|
||||||
|
# revert. ondelete SET NULL rather than CASCADE: deleting an artist must
|
||||||
|
# not destroy the record of where their files were moved.
|
||||||
|
artist_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
planned_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=0, server_default="0",
|
||||||
|
)
|
||||||
|
moved_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=0, server_default="0",
|
||||||
|
)
|
||||||
|
refused_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=0, server_default="0",
|
||||||
|
)
|
||||||
|
|
||||||
|
# [{"image_id": int, "from": str, "to": str}, ...] — see the module
|
||||||
|
# docstring. This is the plan, the audit trail and the undo, in that order
|
||||||
|
# of appearance and in one place.
|
||||||
|
moves: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb"),
|
||||||
|
)
|
||||||
|
# [{"image_id": int, "reason": str}, ...] — rows the apply declined to
|
||||||
|
# touch, with why. A refusal is an expected outcome, not an error: the
|
||||||
|
# world moves between plan and apply, and every gate fails closed.
|
||||||
|
refusals: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb"),
|
||||||
|
)
|
||||||
|
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -16,17 +16,23 @@ writing their own — the house shape for rule 93, snippet #3087. A preview
|
|||||||
that computes its set differently from the apply is a preview that can lie,
|
that computes its set differently from the apply is a preview that can lie,
|
||||||
and here the apply RENAMES the operator's art.
|
and here the apply RENAMES the operator's art.
|
||||||
|
|
||||||
Nothing in this module writes. It reads rows, and it stats files when asked.
|
## What writes and what does not
|
||||||
|
|
||||||
|
`survey_layout` and `plan_placement` are read-only — they report and they
|
||||||
|
record a plan. `apply_run` and `revert_run` are the only functions here that
|
||||||
|
rename a file or rewrite a row, and each does both for one row at a time,
|
||||||
|
updating the row only after its rename lands.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import Artist, ImageRecord
|
from ..models import Artist, ImageRecord, LibraryPlacementRun
|
||||||
from ..utils.paths import canonical_subdir
|
from ..utils.paths import canonical_subdir
|
||||||
|
|
||||||
# Top-level directories under the images root that are STORES, not artists.
|
# Top-level directories under the images root that are STORES, not artists.
|
||||||
@@ -229,3 +235,200 @@ def survey_layout(
|
|||||||
report.unmovable += layout.unmovable
|
report.unmovable += layout.unmovable
|
||||||
|
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
# --- the reconciler: plan -> apply -> revert (#4246) -------------------------
|
||||||
|
#
|
||||||
|
# The three verbs share one materialised plan rather than each deriving its
|
||||||
|
# own set. `plan_placement` writes `LibraryPlacementRun.moves`; `apply_run`
|
||||||
|
# executes THAT list; `revert_run` walks it backwards. A preview that can
|
||||||
|
# disagree with its apply is the failure this shape exists to prevent, and
|
||||||
|
# here the apply renames the operator's art.
|
||||||
|
#
|
||||||
|
# Every step fails CLOSED. The world moves between planning and applying —
|
||||||
|
# a download lands, a supersede rewrites a path, a file is deleted — so the
|
||||||
|
# apply re-checks each row against what the plan recorded and declines the
|
||||||
|
# ones that moved on, instead of trusting a plan that may be minutes old.
|
||||||
|
|
||||||
|
|
||||||
|
def plan_placement(
|
||||||
|
session: Session, images_root: Path, *, artist_id: int | None = None,
|
||||||
|
) -> LibraryPlacementRun:
|
||||||
|
"""Build (and persist) the move plan. Touches no files.
|
||||||
|
|
||||||
|
`artist_id` scopes the run to one artist, which is how this is meant to be
|
||||||
|
used: do one, look at the gallery, then continue or revert. None plans the
|
||||||
|
whole library.
|
||||||
|
"""
|
||||||
|
stmt = select(Artist).order_by(Artist.slug)
|
||||||
|
if artist_id is not None:
|
||||||
|
stmt = stmt.where(Artist.id == artist_id)
|
||||||
|
artists = session.execute(stmt).scalars().all()
|
||||||
|
|
||||||
|
candidates: list[dict] = []
|
||||||
|
wanted: dict[str, int] = {}
|
||||||
|
for artist in artists:
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.path)
|
||||||
|
.where(*_misplaced_conditions(images_root, artist.id, artist.slug))
|
||||||
|
).all()
|
||||||
|
for row_id, path in rows:
|
||||||
|
dest = destination_for(path, images_root, artist.slug)
|
||||||
|
if dest is None or dest.exists():
|
||||||
|
continue
|
||||||
|
key = str(dest)
|
||||||
|
candidates.append({"image_id": row_id, "from": path, "to": key})
|
||||||
|
wanted[key] = wanted.get(key, 0) + 1
|
||||||
|
|
||||||
|
# Two rows wanting one destination: plan NEITHER. Which of them "wins" is
|
||||||
|
# not this sweep's call, and planning one of them would silently pick a
|
||||||
|
# winner by iteration order. Counting first and filtering after is what
|
||||||
|
# makes that true — claiming as we go would quietly keep whichever came
|
||||||
|
# first.
|
||||||
|
moves = [m for m in candidates if wanted[m["to"]] == 1]
|
||||||
|
|
||||||
|
run = LibraryPlacementRun(
|
||||||
|
status="ready", artist_id=artist_id, moves=moves,
|
||||||
|
planned_count=len(moves),
|
||||||
|
)
|
||||||
|
session.add(run)
|
||||||
|
session.flush()
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def _move_one(src: Path, dest: Path) -> str | None:
|
||||||
|
"""Rename `src` to `dest`. Returns a refusal reason, or None on success.
|
||||||
|
|
||||||
|
A rename within one filesystem, so no copy and no free space needed. The
|
||||||
|
destination check is not a race-free guarantee — nothing here is — but it
|
||||||
|
turns the common case of "something already landed there" into a refusal
|
||||||
|
instead of a silent overwrite.
|
||||||
|
"""
|
||||||
|
if not src.exists():
|
||||||
|
return "source missing"
|
||||||
|
if dest.exists():
|
||||||
|
return "destination occupied"
|
||||||
|
try:
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
src.rename(dest)
|
||||||
|
except OSError as exc:
|
||||||
|
return f"rename failed: {exc}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def apply_run(
|
||||||
|
session: Session, run: LibraryPlacementRun, *, chunk: int = 0,
|
||||||
|
) -> LibraryPlacementRun:
|
||||||
|
"""Execute a `ready` run's stored plan: file and row together, per row.
|
||||||
|
|
||||||
|
The row is updated ONLY after its rename lands, so a refused or failed
|
||||||
|
move can never leave `ImageRecord.path` pointing at a file that is not
|
||||||
|
there. Refusals are recorded and the run continues — one row that moved
|
||||||
|
on since planning is not a reason to abandon the other 33,788.
|
||||||
|
|
||||||
|
`chunk` commits progress every N moves. Set it for any real run: the
|
||||||
|
ledger is the ONLY record of where a file came from, so a worker that
|
||||||
|
dies at row 30,000 of 33,789 must not take the undo information for the
|
||||||
|
first 29,999 with it. Left at 0 (tests, small runs) everything persists
|
||||||
|
in one go at the end.
|
||||||
|
|
||||||
|
Re-running a partially-applied plan is safe rather than clever: the rows
|
||||||
|
already moved no longer match their `from`, so they refuse as "row moved
|
||||||
|
since planning" instead of being moved twice.
|
||||||
|
"""
|
||||||
|
if run.status != "ready":
|
||||||
|
raise ValueError(f"run {run.id} is {run.status}, not ready")
|
||||||
|
|
||||||
|
refusals: list[dict] = []
|
||||||
|
moved: list[dict] = []
|
||||||
|
|
||||||
|
def _persist() -> None:
|
||||||
|
# Reassign rather than mutate: SQLAlchemy does not track in-place
|
||||||
|
# changes to a JSONB list, so an .append() alone would never reach
|
||||||
|
# the database and the ledger would silently stay empty.
|
||||||
|
run.moves = list(moved)
|
||||||
|
run.refusals = list(refusals)
|
||||||
|
run.moved_count = len(moved)
|
||||||
|
run.refused_count = len(refusals)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
# Snapshot the plan before iterating: `_persist` reassigns `run.moves`,
|
||||||
|
# and iterating the attribute while rewriting it would walk a list that
|
||||||
|
# changes underneath the loop.
|
||||||
|
plan = list(run.moves)
|
||||||
|
for done, move in enumerate(plan, start=1):
|
||||||
|
record = session.get(ImageRecord, move["image_id"])
|
||||||
|
if record is None:
|
||||||
|
refusals.append({"image_id": move["image_id"], "reason": "row gone"})
|
||||||
|
continue
|
||||||
|
if record.path != move["from"]:
|
||||||
|
# Something rewrote this row since the plan was built — a
|
||||||
|
# supersede, or an earlier run. The plan is stale for it.
|
||||||
|
refusals.append({
|
||||||
|
"image_id": move["image_id"], "reason": "row moved since planning",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
reason = _move_one(Path(move["from"]), Path(move["to"]))
|
||||||
|
if reason is not None:
|
||||||
|
refusals.append({"image_id": move["image_id"], "reason": reason})
|
||||||
|
continue
|
||||||
|
record.path = move["to"]
|
||||||
|
moved.append(move)
|
||||||
|
if chunk and done % chunk == 0:
|
||||||
|
_persist()
|
||||||
|
|
||||||
|
run.moves = moved
|
||||||
|
run.refusals = refusals
|
||||||
|
run.moved_count = len(moved)
|
||||||
|
run.refused_count = len(refusals)
|
||||||
|
run.status = "applied"
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.flush()
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def revert_run(
|
||||||
|
session: Session, run: LibraryPlacementRun,
|
||||||
|
) -> LibraryPlacementRun:
|
||||||
|
"""Put an applied run's files back where they came from.
|
||||||
|
|
||||||
|
This is why `from` is retained. It is the answer to "do one artist, look
|
||||||
|
at it, and undo if it reads wrong" — which is a cheaper way to settle
|
||||||
|
whether artist_id or the folder held the truth (#4257) than arguing it
|
||||||
|
from a sample.
|
||||||
|
|
||||||
|
Refuses the same way the apply does: a file someone has since moved or
|
||||||
|
replaced stays where it is, and its row is left alone.
|
||||||
|
|
||||||
|
A revert interrupted half way is resumable by re-running it: the rows
|
||||||
|
already put back no longer sit at `to`, so they refuse rather than move
|
||||||
|
twice. Unlike the apply this needs no chunked persistence — it consumes
|
||||||
|
the ledger rather than producing it, so a crash costs progress, not
|
||||||
|
information.
|
||||||
|
"""
|
||||||
|
if run.status != "applied":
|
||||||
|
raise ValueError(f"run {run.id} is {run.status}, not applied")
|
||||||
|
|
||||||
|
refusals: list[dict] = []
|
||||||
|
reverted = 0
|
||||||
|
for move in run.moves:
|
||||||
|
record = session.get(ImageRecord, move["image_id"])
|
||||||
|
if record is None or record.path != move["to"]:
|
||||||
|
refusals.append({
|
||||||
|
"image_id": move["image_id"], "reason": "row changed since apply",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
reason = _move_one(Path(move["to"]), Path(move["from"]))
|
||||||
|
if reason is not None:
|
||||||
|
refusals.append({"image_id": move["image_id"], "reason": reason})
|
||||||
|
continue
|
||||||
|
record.path = move["from"]
|
||||||
|
reverted += 1
|
||||||
|
|
||||||
|
run.refusals = refusals
|
||||||
|
run.refused_count = len(refusals)
|
||||||
|
run.moved_count = run.moved_count - reverted
|
||||||
|
run.status = "reverted"
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.flush()
|
||||||
|
return run
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
"""Placement reconciler tasks — plan, apply, revert (milestone #421).
|
||||||
|
|
||||||
|
The service (`services.library_layout`) holds the decisions; this module is
|
||||||
|
only the async wrapper, matching `tasks.library_audit`: run on the
|
||||||
|
maintenance queue, mark the run `error` with a traceback if anything escapes,
|
||||||
|
and return a small summary dict so eager-mode tests can assert on it.
|
||||||
|
|
||||||
|
Applying is a long run — 33,789 renames on the operator's library at the time
|
||||||
|
of writing — so `apply_placement` persists its ledger in chunks rather than
|
||||||
|
at the end. That ledger is the only record of where each file came from, and
|
||||||
|
a worker that dies two thirds of the way through must not take the undo
|
||||||
|
information for the first two thirds with it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import traceback
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..models import LibraryPlacementRun
|
||||||
|
from ..services import library_layout
|
||||||
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
# Commit the ledger every this many moves. Small enough that a crash loses
|
||||||
|
# seconds of work, large enough not to make a COMMIT per rename.
|
||||||
|
_APPLY_CHUNK = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _fail(session, run_id: int, message: str) -> None:
|
||||||
|
run = session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is not None:
|
||||||
|
run.status = "error"
|
||||||
|
run.error = message
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.library_placement.plan_placement",
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=5, retry_backoff_max=60, retry_jitter=True, max_retries=3,
|
||||||
|
soft_time_limit=900, time_limit=1000,
|
||||||
|
)
|
||||||
|
def plan_placement(artist_id: int | None = None) -> dict:
|
||||||
|
"""Build a move plan and leave it `ready` for the operator to read.
|
||||||
|
|
||||||
|
Reads rows and stats destinations; moves nothing.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
run = library_layout.plan_placement(
|
||||||
|
session, IMAGES_ROOT, artist_id=artist_id,
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {
|
||||||
|
"run_id": run.id, "status": run.status,
|
||||||
|
"planned_count": run.planned_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.library_placement.apply_placement",
|
||||||
|
soft_time_limit=7200, time_limit=7500,
|
||||||
|
)
|
||||||
|
def apply_placement(run_id: int) -> dict:
|
||||||
|
"""Execute a `ready` run's stored plan. Renames files and rewrites rows.
|
||||||
|
|
||||||
|
No autoretry: a retry would re-enter a half-applied plan on a schedule
|
||||||
|
nobody asked for. Re-running IS safe (the applied rows refuse as "row
|
||||||
|
moved since planning"), but that should be the operator's decision after
|
||||||
|
reading what happened, not the queue's.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
run = session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return {"run_id": run_id, "status": "missing"}
|
||||||
|
if run.status != "ready":
|
||||||
|
return {"run_id": run_id, "status": run.status, "skipped": True}
|
||||||
|
try:
|
||||||
|
library_layout.apply_run(session, run, chunk=_APPLY_CHUNK)
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
log.exception("placement apply failed for run %s", run_id)
|
||||||
|
session.rollback()
|
||||||
|
_fail(session, run_id, traceback.format_exc())
|
||||||
|
return {"run_id": run_id, "status": "error"}
|
||||||
|
return {
|
||||||
|
"run_id": run_id, "status": run.status,
|
||||||
|
"moved": run.moved_count, "refused": run.refused_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.library_placement.revert_placement",
|
||||||
|
soft_time_limit=7200, time_limit=7500,
|
||||||
|
)
|
||||||
|
def revert_placement(run_id: int) -> dict:
|
||||||
|
"""Put an applied run's files back where they came from."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
run = session.get(LibraryPlacementRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return {"run_id": run_id, "status": "missing"}
|
||||||
|
if run.status != "applied":
|
||||||
|
return {"run_id": run_id, "status": run.status, "skipped": True}
|
||||||
|
try:
|
||||||
|
library_layout.revert_run(session, run)
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
log.exception("placement revert failed for run %s", run_id)
|
||||||
|
session.rollback()
|
||||||
|
_fail(session, run_id, traceback.format_exc())
|
||||||
|
return {"run_id": run_id, "status": "error"}
|
||||||
|
return {
|
||||||
|
"run_id": run_id, "status": run.status,
|
||||||
|
"refused": run.refused_count,
|
||||||
|
}
|
||||||
@@ -54,6 +54,7 @@
|
|||||||
Self-healing and repair: missing files, thumbnails, database upkeep.
|
Self-healing and repair: missing files, thumbnails, database upkeep.
|
||||||
</p>
|
</p>
|
||||||
<div class="fc-tile-grid">
|
<div class="fc-tile-grid">
|
||||||
|
<PlacementCard />
|
||||||
<MissingFileRepairCard />
|
<MissingFileRepairCard />
|
||||||
<ThumbnailBackfillCard />
|
<ThumbnailBackfillCard />
|
||||||
<DbMaintenanceCard />
|
<DbMaintenanceCard />
|
||||||
@@ -80,6 +81,7 @@ import MLBackfillCard from './MLBackfillCard.vue'
|
|||||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||||
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
||||||
|
import PlacementCard from './PlacementCard.vue'
|
||||||
import GpuTriageCard from './GpuTriageCard.vue'
|
import GpuTriageCard from './GpuTriageCard.vue'
|
||||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||||
import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
<template>
|
||||||
|
<MaintenanceTile
|
||||||
|
icon="mdi-folder-move"
|
||||||
|
title="File placement"
|
||||||
|
blurb="Put every image in its own artist's folder, one artist at a time."
|
||||||
|
>
|
||||||
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
|
The library keeps one folder per artist, named after them. Files written
|
||||||
|
under older rules can sit in another artist's folder — this moves them
|
||||||
|
home, updating the record and the file together. Every run can be
|
||||||
|
reverted, so the safe way to use it is one artist at a time: run it,
|
||||||
|
look at the gallery, then continue or put it back.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="error" type="warning" variant="tonal" density="compact" class="mb-3"
|
||||||
|
>{{ error }}</v-alert>
|
||||||
|
|
||||||
|
<!-- Survey -->
|
||||||
|
<div class="fc-settings-row mb-3">
|
||||||
|
<v-btn
|
||||||
|
variant="tonal" rounded="pill" prepend-icon="mdi-magnify"
|
||||||
|
:loading="surveying" @click="onSurvey"
|
||||||
|
>Check placement</v-btn>
|
||||||
|
<span v-if="layout" class="fc-muted text-body-2">
|
||||||
|
{{ layout.misplaced_rows.toLocaleString() }} of
|
||||||
|
{{ layout.total_rows.toLocaleString() }} images are in the wrong folder
|
||||||
|
<template v-if="layout.misplaced_rows">
|
||||||
|
— across {{ layout.artists.length }} artists
|
||||||
|
</template>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-table v-if="layout && layout.artists.length" density="compact" class="mb-2">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Artist</th>
|
||||||
|
<th class="text-right">To move</th>
|
||||||
|
<th>Currently in</th>
|
||||||
|
<th class="text-right">Plan</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="a in layout.artists" :key="a.artist_id">
|
||||||
|
<td>{{ a.name }}</td>
|
||||||
|
<td class="text-right fc-tabular">{{ a.misplaced_rows.toLocaleString() }}</td>
|
||||||
|
<td class="fc-muted text-body-2">{{ a.stray_dirs.join(', ') }}</td>
|
||||||
|
<td class="text-right">
|
||||||
|
<v-btn
|
||||||
|
size="small" variant="text" :loading="planningId === a.artist_id"
|
||||||
|
@click="onPlan(a.artist_id)"
|
||||||
|
>Plan</v-btn>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
|
||||||
|
<v-divider class="my-4" />
|
||||||
|
|
||||||
|
<!-- Runs -->
|
||||||
|
<h3 class="fc-section-title">Runs</h3>
|
||||||
|
<v-table density="compact">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>When</th>
|
||||||
|
<th>Scope</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th class="text-right">Planned</th>
|
||||||
|
<th class="text-right">Moved</th>
|
||||||
|
<th class="text-right">Refused</th>
|
||||||
|
<th class="text-right">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="r in store.placementRuns" :key="r.id">
|
||||||
|
<td class="fc-tabular" :title="r.started_at">
|
||||||
|
{{ formatRelative(r.started_at) }}
|
||||||
|
</td>
|
||||||
|
<td>{{ artistName(r.artist_id) }}</td>
|
||||||
|
<td>
|
||||||
|
<v-icon size="small" :color="statusColor(r.status)">
|
||||||
|
{{ statusIcon(r.status) }}
|
||||||
|
</v-icon>
|
||||||
|
{{ r.status }}
|
||||||
|
</td>
|
||||||
|
<td class="text-right fc-tabular">{{ r.planned_count.toLocaleString() }}</td>
|
||||||
|
<td class="text-right fc-tabular">{{ r.moved_count.toLocaleString() }}</td>
|
||||||
|
<td class="text-right fc-tabular">
|
||||||
|
<span :class="r.refused_count ? 'text-warning' : 'fc-muted'">
|
||||||
|
{{ r.refused_count.toLocaleString() }}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td class="text-right">
|
||||||
|
<KebabMenu :label="`Actions for placement run ${r.id}`">
|
||||||
|
<v-list-item title="Review…" @click="onReview(r)" />
|
||||||
|
<v-list-item
|
||||||
|
title="Move files" :disabled="r.status !== 'ready' || !r.planned_count"
|
||||||
|
@click="onApply(r)"
|
||||||
|
/>
|
||||||
|
<v-list-item
|
||||||
|
title="Put back…" :disabled="r.status !== 'applied'"
|
||||||
|
@click="onRevert(r)"
|
||||||
|
/>
|
||||||
|
</KebabMenu>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr v-if="!store.placementRuns.length">
|
||||||
|
<td colspan="7" class="text-center fc-muted py-4">
|
||||||
|
No runs yet. Check placement above, then plan one artist.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
|
||||||
|
<!-- Review: the plan, which is what gets agreed to -->
|
||||||
|
<v-dialog v-model="reviewOpen" max-width="900">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>
|
||||||
|
Run {{ reviewRun?.id }} — {{ reviewRun?.planned_count?.toLocaleString() }} moves
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Showing the first {{ REVIEW_LIMIT }}. Each row moves the file and
|
||||||
|
its record together; nothing is overwritten.
|
||||||
|
</p>
|
||||||
|
<v-table density="compact">
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="m in reviewMoves" :key="m.image_id">
|
||||||
|
<td class="fc-muted text-caption">{{ m.from }}</td>
|
||||||
|
<td class="text-caption">→ {{ m.to }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
<div v-if="reviewRun?.refusals?.length" class="mt-4">
|
||||||
|
<h4 class="fc-section-title">Refused ({{ reviewRun.refusals.length }})</h4>
|
||||||
|
<p class="fc-muted text-body-2">
|
||||||
|
Rows the run declined to touch — the source moved, the
|
||||||
|
destination was taken, or the record changed since planning.
|
||||||
|
</p>
|
||||||
|
<div
|
||||||
|
v-for="f in reviewRun.refusals.slice(0, REVIEW_LIMIT)" :key="f.image_id"
|
||||||
|
class="text-caption fc-muted"
|
||||||
|
>#{{ f.image_id }} — {{ f.reason }}</div>
|
||||||
|
</div>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="reviewOpen = false">Close</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
|
||||||
|
<v-dialog v-model="confirmOpen" max-width="520">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>{{ confirmTitle }}</v-card-title>
|
||||||
|
<v-card-text class="text-body-2">{{ confirmMessage }}</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
@click="confirmOpen = false; onConfirmed()"
|
||||||
|
>Go ahead</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</MaintenanceTile>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
import { formatRelative } from '../../utils/date.js'
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
|
|
||||||
|
// A whole-library plan is tens of thousands of moves. The dialog is for
|
||||||
|
// judging whether the moves look right, which a sample answers — the full
|
||||||
|
// list is in the run row if anyone needs it.
|
||||||
|
const REVIEW_LIMIT = 200
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const layout = ref(null)
|
||||||
|
const artists = ref({})
|
||||||
|
const surveying = ref(false)
|
||||||
|
const planningId = ref(null)
|
||||||
|
const error = ref('')
|
||||||
|
|
||||||
|
const reviewOpen = ref(false)
|
||||||
|
const reviewRun = ref(null)
|
||||||
|
const reviewMoves = computed(() => (reviewRun.value?.moves || []).slice(0, REVIEW_LIMIT))
|
||||||
|
|
||||||
|
const confirmOpen = ref(false)
|
||||||
|
const confirmTitle = ref('')
|
||||||
|
const confirmMessage = ref('')
|
||||||
|
let confirmAction = null
|
||||||
|
|
||||||
|
let poll = null
|
||||||
|
|
||||||
|
function artistName(id) {
|
||||||
|
if (id === null || id === undefined) return 'Whole library'
|
||||||
|
return artists.value[id] || `#${id}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusIcon(s) {
|
||||||
|
return {
|
||||||
|
ready: 'mdi-clipboard-text-outline', applied: 'mdi-check-circle',
|
||||||
|
reverted: 'mdi-undo-variant', error: 'mdi-close-circle',
|
||||||
|
running: 'mdi-timer-sand', cancelled: 'mdi-cancel',
|
||||||
|
}[s] || 'mdi-help-circle'
|
||||||
|
}
|
||||||
|
function statusColor(s) {
|
||||||
|
return {
|
||||||
|
ready: 'accent', applied: 'success', reverted: 'info',
|
||||||
|
error: 'error', running: 'warning',
|
||||||
|
}[s] || 'grey'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function guard(fn) {
|
||||||
|
error.value = ''
|
||||||
|
try {
|
||||||
|
return await fn()
|
||||||
|
} catch (e) {
|
||||||
|
error.value = e?.message || String(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onSurvey() {
|
||||||
|
surveying.value = true
|
||||||
|
try {
|
||||||
|
await guard(async () => { layout.value = await store.loadLayout() })
|
||||||
|
} finally {
|
||||||
|
surveying.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPlan(artistId) {
|
||||||
|
planningId.value = artistId
|
||||||
|
try {
|
||||||
|
await guard(async () => {
|
||||||
|
await store.planPlacement(artistId)
|
||||||
|
toast('Planning — the run appears below when it is ready')
|
||||||
|
await store.loadPlacementRuns()
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
planningId.value = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onReview(run) {
|
||||||
|
await guard(async () => {
|
||||||
|
reviewRun.value = await store.getPlacementRun(run.id)
|
||||||
|
reviewOpen.value = true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function ask(title, message, action) {
|
||||||
|
confirmTitle.value = title
|
||||||
|
confirmMessage.value = message
|
||||||
|
confirmAction = action
|
||||||
|
confirmOpen.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
function onApply(run) {
|
||||||
|
ask(
|
||||||
|
'Move these files?',
|
||||||
|
`${run.planned_count.toLocaleString()} files move into their artist's `
|
||||||
|
+ 'folder. The record and the file move together, and this run can be '
|
||||||
|
+ 'put back afterwards.',
|
||||||
|
async () => {
|
||||||
|
await store.applyPlacement(run.id)
|
||||||
|
toast('Moving files — this runs in the background')
|
||||||
|
await store.loadPlacementRuns()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function onRevert(run) {
|
||||||
|
ask(
|
||||||
|
'Put these files back?',
|
||||||
|
`${run.moved_count.toLocaleString()} files return to where they were `
|
||||||
|
+ 'before this run.',
|
||||||
|
async () => {
|
||||||
|
await store.revertPlacement(run.id)
|
||||||
|
toast('Putting files back — this runs in the background')
|
||||||
|
await store.loadPlacementRuns()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmed() {
|
||||||
|
const action = confirmAction
|
||||||
|
confirmAction = null
|
||||||
|
if (action) await guard(action)
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await guard(async () => {
|
||||||
|
artists.value = await store.loadArtistNames()
|
||||||
|
await store.loadPlacementRuns()
|
||||||
|
})
|
||||||
|
// Runs change state in a worker, so the table needs refreshing. The rows
|
||||||
|
// are server-side, which is why no localStorage resurfacing is needed here
|
||||||
|
// (useMaintenanceTask's job) — a reload or another machine sees the same
|
||||||
|
// state because the state IS the row.
|
||||||
|
poll = setInterval(() => { store.loadPlacementRuns().catch(() => {}) }, 5000)
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => { if (poll) clearInterval(poll) })
|
||||||
|
</script>
|
||||||
@@ -71,10 +71,66 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
|||||||
return await api.post(`/api/cleanup/audit/${id}/cancel`)
|
return await api.post(`/api/cleanup/audit/${id}/cancel`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- placement reconciler (milestone #421) --------------------------------
|
||||||
|
//
|
||||||
|
// Runs are server-side rows, so the DATABASE is the durable state here —
|
||||||
|
// no localStorage resurfacing (useMaintenanceTask) is needed. Reload the
|
||||||
|
// page, open it on another machine, and the run and its status are simply
|
||||||
|
// there. That also means a plan survives being walked away from for a day.
|
||||||
|
|
||||||
|
const placementRuns = ref([])
|
||||||
|
const layout = ref(null)
|
||||||
|
|
||||||
|
// The survey: which rows sit outside their artist's directory. check_disk
|
||||||
|
// additionally stats every destination (collisions, missing sources) and
|
||||||
|
// costs one stat per misplaced row over NFS, so it is opt-in.
|
||||||
|
async function loadLayout(checkDisk = false) {
|
||||||
|
layout.value = await api.get('/api/cleanup/layout', {
|
||||||
|
params: checkDisk ? { check_disk: 1 } : {},
|
||||||
|
})
|
||||||
|
return layout.value
|
||||||
|
}
|
||||||
|
|
||||||
|
// id -> name, so a run row can say "Conto" instead of "#47". The runs
|
||||||
|
// endpoint carries artist_id alone: the name belongs to the artist, and
|
||||||
|
// denormalising it into every run would go stale the moment one is renamed.
|
||||||
|
async function loadArtistNames() {
|
||||||
|
const rows = await api.get('/api/artists/names')
|
||||||
|
return Object.fromEntries((rows || []).map(a => [a.id, a.name]))
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadPlacementRuns(limit = 25) {
|
||||||
|
const body = await api.get('/api/cleanup/placement/runs', { params: { limit } })
|
||||||
|
placementRuns.value = body.runs || []
|
||||||
|
return placementRuns.value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detail carries `moves` — the plan the operator reads before agreeing.
|
||||||
|
async function getPlacementRun(id) {
|
||||||
|
return await api.get(`/api/cleanup/placement/runs/${id}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function planPlacement(artistId = null) {
|
||||||
|
return await api.post('/api/cleanup/placement/plan', {
|
||||||
|
body: artistId === null ? {} : { artist_id: artistId },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyPlacement(id) {
|
||||||
|
return await api.post(`/api/cleanup/placement/runs/${id}/apply`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function revertPlacement(id) {
|
||||||
|
return await api.post(`/api/cleanup/placement/runs/${id}/revert`)
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
defaults, recentRuns,
|
defaults, recentRuns,
|
||||||
loadDefaults,
|
loadDefaults,
|
||||||
previewMinDim, deleteMinDim,
|
previewMinDim, deleteMinDim,
|
||||||
startAudit, getAudit, loadHistory, latestAuditForRule, applyAudit, cancelAudit,
|
startAudit, getAudit, loadHistory, latestAuditForRule, applyAudit, cancelAudit,
|
||||||
|
placementRuns, layout,
|
||||||
|
loadLayout, loadArtistNames, loadPlacementRuns, getPlacementRun,
|
||||||
|
planPlacement, applyPlacement, revertPlacement,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
import { setActivePinia, createPinia } from 'pinia'
|
||||||
|
import { useCleanupStore } from '../src/stores/cleanup.js'
|
||||||
|
import { stubFetch } from './stubFetch.js'
|
||||||
|
|
||||||
|
|
||||||
|
describe('placement reconciler store (milestone #421)', () => {
|
||||||
|
beforeEach(() => setActivePinia(createPinia()))
|
||||||
|
afterEach(() => vi.restoreAllMocks())
|
||||||
|
|
||||||
|
it('loadLayout leaves the disk check off by default', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
let seen = ''
|
||||||
|
stubFetch((url) => {
|
||||||
|
seen = url
|
||||||
|
return { status: 200, body: { total_rows: 10, misplaced_rows: 2, artists: [] } }
|
||||||
|
})
|
||||||
|
await s.loadLayout()
|
||||||
|
// One stat per misplaced row over NFS is the cost; it must be opt-in.
|
||||||
|
expect(seen).not.toContain('check_disk')
|
||||||
|
expect(s.layout.misplaced_rows).toBe(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loadLayout asks for the disk check when requested', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
let seen = ''
|
||||||
|
stubFetch((url) => {
|
||||||
|
seen = url
|
||||||
|
return { status: 200, body: { total_rows: 0, misplaced_rows: 0, artists: [] } }
|
||||||
|
})
|
||||||
|
await s.loadLayout(true)
|
||||||
|
expect(seen).toContain('check_disk=1')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('planPlacement scopes to an artist when given one', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
let sent = null
|
||||||
|
stubFetch((url, init) => {
|
||||||
|
sent = JSON.parse(init.body)
|
||||||
|
return { status: 202, body: { status: 'dispatched' } }
|
||||||
|
})
|
||||||
|
await s.planPlacement(47)
|
||||||
|
expect(sent).toEqual({ artist_id: 47 })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('planPlacement sends no scope for the whole library', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
let sent = null
|
||||||
|
stubFetch((url, init) => {
|
||||||
|
sent = JSON.parse(init.body)
|
||||||
|
return { status: 202, body: { status: 'dispatched' } }
|
||||||
|
})
|
||||||
|
await s.planPlacement()
|
||||||
|
// Not `{artist_id: null}` — the endpoint rejects a non-integer, and an
|
||||||
|
// absent key is how "whole library" is spelled.
|
||||||
|
expect(sent).toEqual({})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loadPlacementRuns keeps the rows for the table', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
stubFetch(() => ({
|
||||||
|
status: 200,
|
||||||
|
body: { runs: [{ id: 3, status: 'ready', planned_count: 12 }] },
|
||||||
|
}))
|
||||||
|
await s.loadPlacementRuns()
|
||||||
|
expect(s.placementRuns).toHaveLength(1)
|
||||||
|
expect(s.placementRuns[0].status).toBe('ready')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('getPlacementRun carries the moves — it is the preview', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
stubFetch(() => ({
|
||||||
|
status: 200,
|
||||||
|
body: {
|
||||||
|
id: 3, status: 'ready', planned_count: 1,
|
||||||
|
moves: [{ image_id: 9, from: '/images/Conto/x.png', to: '/images/conto/x.png' }],
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
const run = await s.getPlacementRun(3)
|
||||||
|
expect(run.moves[0].from).toBe('/images/Conto/x.png')
|
||||||
|
expect(run.moves[0].to).toBe('/images/conto/x.png')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loadArtistNames maps id to name for the run rows', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
stubFetch(() => ({
|
||||||
|
status: 200,
|
||||||
|
body: [{ id: 47, name: 'Conto', slug: 'conto' }],
|
||||||
|
}))
|
||||||
|
expect(await s.loadArtistNames()).toEqual({ 47: 'Conto' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loadArtistNames survives an empty roster', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
stubFetch(() => ({ status: 200, body: [] }))
|
||||||
|
expect(await s.loadArtistNames()).toEqual({})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applyPlacement and revertPlacement post to their own run', async () => {
|
||||||
|
const s = useCleanupStore()
|
||||||
|
const urls = []
|
||||||
|
stubFetch((url) => {
|
||||||
|
urls.push(url)
|
||||||
|
return { status: 202, body: { status: 'dispatched' } }
|
||||||
|
})
|
||||||
|
await s.applyPlacement(5)
|
||||||
|
await s.revertPlacement(5)
|
||||||
|
expect(urls[0]).toContain('/api/cleanup/placement/runs/5/apply')
|
||||||
|
expect(urls[1]).toContain('/api/cleanup/placement/runs/5/revert')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { vi } from 'vitest'
|
||||||
|
|
||||||
|
// The canonical fetch stub for store specs.
|
||||||
|
//
|
||||||
|
// `handler(url, init)` returns `{ status, body }`; body is JSON-encoded, and
|
||||||
|
// `ok` is derived from the status so a store's error path can be exercised by
|
||||||
|
// returning 4xx/5xx. Returns the vi.fn so a caller can assert on calls.
|
||||||
|
//
|
||||||
|
// Extracted 2026-09-21 from six specs carrying byte-identical copies
|
||||||
|
// (adminStore, credentials, dbMaintenance, gallery, suggestions,
|
||||||
|
// galleryRelatedStrip). Those still hold their own; migrate each the next
|
||||||
|
// time it is touched rather than in one sweep.
|
||||||
|
export function stubFetch (handler) {
|
||||||
|
globalThis.fetch = vi.fn(async (url, init) => {
|
||||||
|
const { status, body } = handler(url, init)
|
||||||
|
return {
|
||||||
|
ok: status >= 200 && status < 300,
|
||||||
|
status,
|
||||||
|
statusText: String(status),
|
||||||
|
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return globalThis.fetch
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
"""The placement reconciler's task + API surface (milestone #421, slice 3b).
|
||||||
|
|
||||||
|
The move logic itself is covered in tests/test_library_layout.py; this module
|
||||||
|
covers the wrapper — that the tasks are registered and routed, that the
|
||||||
|
endpoints gate on run state, and that a list response stays small.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
import backend.app.tasks.library_placement # noqa: F401 — register tasks
|
||||||
|
from backend.app.celery_app import celery
|
||||||
|
from backend.app.models import Artist, LibraryPlacementRun
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
_TASKS = (
|
||||||
|
"backend.app.tasks.library_placement.plan_placement",
|
||||||
|
"backend.app.tasks.library_placement.apply_placement",
|
||||||
|
"backend.app.tasks.library_placement.revert_placement",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name", _TASKS)
|
||||||
|
def test_placement_tasks_are_registered(name):
|
||||||
|
assert name in celery.tasks
|
||||||
|
|
||||||
|
|
||||||
|
def test_placement_runs_on_the_long_maintenance_lane():
|
||||||
|
"""33k renames must not sit in the quick lane, which is where the
|
||||||
|
self-healing sweeps live (the 2026-06-07 starvation)."""
|
||||||
|
routes = celery.conf.task_routes
|
||||||
|
assert routes["backend.app.tasks.library_placement.*"] == {
|
||||||
|
"queue": "maintenance_long"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db, status="ready", moves=None, artist_id=None):
|
||||||
|
"""Adds the row; the caller awaits the COMMIT.
|
||||||
|
|
||||||
|
Commit, not flush: the app under test runs on its own session and
|
||||||
|
connection, so a flush that stays inside this test's transaction is
|
||||||
|
invisible to the endpoint — the row simply is not there yet. Same reason
|
||||||
|
`_seed_runs` in test_api_system_backup commits."""
|
||||||
|
run = LibraryPlacementRun(
|
||||||
|
status=status, artist_id=artist_id, moves=moves or [],
|
||||||
|
planned_count=len(moves or []),
|
||||||
|
)
|
||||||
|
db.add(run)
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runs_list_omits_the_moves(client, db):
|
||||||
|
"""An applied whole-library run carries tens of thousands of entries.
|
||||||
|
Fine in Postgres, wrong in every list response."""
|
||||||
|
run = LibraryPlacementRun(
|
||||||
|
status="applied",
|
||||||
|
moves=[{"image_id": 1, "from": "/images/A/x.png", "to": "/images/a/x.png"}],
|
||||||
|
planned_count=1, moved_count=1,
|
||||||
|
)
|
||||||
|
db.add(run)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.get("/api/cleanup/placement/runs")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["runs"][0]["planned_count"] == 1
|
||||||
|
assert "moves" not in body["runs"][0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_detail_carries_the_moves(client, db):
|
||||||
|
"""The detail IS the preview the operator reads before agreeing."""
|
||||||
|
run = LibraryPlacementRun(
|
||||||
|
status="ready",
|
||||||
|
moves=[{"image_id": 7, "from": "/images/Conto/x.png", "to": "/images/conto/x.png"}],
|
||||||
|
planned_count=1,
|
||||||
|
)
|
||||||
|
db.add(run)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.get(f"/api/cleanup/placement/runs/{run.id}")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["moves"][0]["from"] == "/images/Conto/x.png"
|
||||||
|
assert body["moves"][0]["to"] == "/images/conto/x.png"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_detail_404s_for_an_unknown_run(client):
|
||||||
|
resp = await client.get("/api/cleanup/placement/runs/999999")
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_plan_rejects_a_non_integer_artist(client):
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/placement/plan", json={"artist_id": "conto"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["error"] == "invalid_artist_id"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_plan_accepts_an_artist_scope(client, db, monkeypatch):
|
||||||
|
sent = {}
|
||||||
|
from backend.app.tasks import library_placement
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
library_placement.plan_placement, "delay",
|
||||||
|
lambda artist_id=None: sent.update(artist_id=artist_id),
|
||||||
|
)
|
||||||
|
artist = Artist(name="Conto", slug="conto")
|
||||||
|
db.add(artist)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/placement/plan", json={"artist_id": artist.id},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 202
|
||||||
|
assert sent["artist_id"] == artist.id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_apply_refuses_a_run_that_is_not_ready(client, db):
|
||||||
|
"""The gate is here as well as in the service — an applied run must not
|
||||||
|
be re-applied by a stray POST."""
|
||||||
|
run = _run(db, status="applied")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/apply")
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["error"] == "not_ready"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_revert_refuses_a_run_that_was_never_applied(client, db):
|
||||||
|
run = _run(db, status="ready")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/revert")
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["error"] == "not_applied"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_apply_dispatches_for_a_ready_run(client, db, monkeypatch):
|
||||||
|
sent = {}
|
||||||
|
from backend.app.tasks import library_placement
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
library_placement.apply_placement, "delay",
|
||||||
|
lambda run_id: sent.update(run_id=run_id),
|
||||||
|
)
|
||||||
|
run = _run(db, status="ready")
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/cleanup/placement/runs/{run.id}/apply")
|
||||||
|
assert resp.status_code == 202
|
||||||
|
assert sent["run_id"] == run.id
|
||||||
|
# Dispatch only — the endpoint must not have moved anything itself.
|
||||||
|
still = (await db.execute(
|
||||||
|
select(LibraryPlacementRun.status)
|
||||||
|
.where(LibraryPlacementRun.id == run.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert still == "ready"
|
||||||
@@ -213,3 +213,191 @@ def test_survey_is_read_only(db_sync, tmp_path):
|
|||||||
assert db_sync.execute(
|
assert db_sync.execute(
|
||||||
select(ImageRecord.path).where(ImageRecord.id == rec.id)
|
select(ImageRecord.path).where(ImageRecord.id == rec.id)
|
||||||
).scalar_one() == before
|
).scalar_one() == before
|
||||||
|
|
||||||
|
|
||||||
|
# --- plan / apply / revert (#4246) ------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _staged(db, tmp_path, slug, stray, name="x.png", n=100):
|
||||||
|
"""An artist with one file sitting in `stray`'s directory."""
|
||||||
|
artist = _artist(db, slug.title(), slug)
|
||||||
|
src = tmp_path / stray / name
|
||||||
|
src.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
src.write_bytes(b"pixels")
|
||||||
|
rec = _image(db, str(src), artist, n)
|
||||||
|
return artist, rec, src
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_plan_records_where_each_file_came_from(db_sync, tmp_path):
|
||||||
|
from backend.app.services.library_layout import plan_placement
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=20)
|
||||||
|
run = plan_placement(db_sync, tmp_path)
|
||||||
|
|
||||||
|
assert run.status == "ready"
|
||||||
|
assert run.planned_count == 1
|
||||||
|
assert run.moves == [{
|
||||||
|
"image_id": rec.id,
|
||||||
|
"from": str(src),
|
||||||
|
"to": str(tmp_path / "conto" / "x.png"),
|
||||||
|
}]
|
||||||
|
# Planning touches nothing.
|
||||||
|
assert src.exists()
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).path == str(src)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_apply_moves_file_and_row_together(db_sync, tmp_path):
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=21)
|
||||||
|
run = apply_run(db_sync, plan_placement(db_sync, tmp_path))
|
||||||
|
|
||||||
|
dest = tmp_path / "conto" / "x.png"
|
||||||
|
assert run.status == "applied"
|
||||||
|
assert run.moved_count == 1 and run.refused_count == 0
|
||||||
|
assert dest.exists() and not src.exists()
|
||||||
|
db_sync.expire_all()
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).path == str(dest)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_revert_puts_it_back(db_sync, tmp_path):
|
||||||
|
"""The whole reason `from` is retained: do one artist, look, undo."""
|
||||||
|
from backend.app.services.library_layout import (
|
||||||
|
apply_run,
|
||||||
|
plan_placement,
|
||||||
|
revert_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=22)
|
||||||
|
run = revert_run(db_sync, apply_run(db_sync, plan_placement(db_sync, tmp_path)))
|
||||||
|
|
||||||
|
assert run.status == "reverted"
|
||||||
|
assert src.exists()
|
||||||
|
assert not (tmp_path / "conto" / "x.png").exists()
|
||||||
|
db_sync.expire_all()
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).path == str(src)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_apply_refuses_a_row_that_moved_since_planning(db_sync, tmp_path):
|
||||||
|
"""A supersede or an earlier run can rewrite a path between plan and
|
||||||
|
apply. The stale entry is declined, not forced."""
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=23)
|
||||||
|
run = plan_placement(db_sync, tmp_path)
|
||||||
|
|
||||||
|
elsewhere = tmp_path / "conto" / "already-here.png"
|
||||||
|
elsewhere.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
elsewhere.write_bytes(b"pixels")
|
||||||
|
rec.path = str(elsewhere)
|
||||||
|
db_sync.flush()
|
||||||
|
|
||||||
|
run = apply_run(db_sync, run)
|
||||||
|
|
||||||
|
assert run.moved_count == 0 and run.refused_count == 1
|
||||||
|
assert run.refusals[0]["reason"] == "row moved since planning"
|
||||||
|
assert src.exists() # untouched
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_apply_never_overwrites_an_occupied_destination(db_sync, tmp_path):
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=24)
|
||||||
|
run = plan_placement(db_sync, tmp_path)
|
||||||
|
|
||||||
|
squatter = tmp_path / "conto" / "x.png"
|
||||||
|
squatter.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
squatter.write_bytes(b"someone else")
|
||||||
|
|
||||||
|
run = apply_run(db_sync, run)
|
||||||
|
|
||||||
|
assert run.refused_count == 1
|
||||||
|
assert run.refusals[0]["reason"] == "destination occupied"
|
||||||
|
assert squatter.read_bytes() == b"someone else"
|
||||||
|
db_sync.expire_all()
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).path == str(src)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_apply_leaves_the_row_alone_when_the_source_is_gone(db_sync, tmp_path):
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
_, rec, src = _staged(db_sync, tmp_path, "conto", "Conto", n=25)
|
||||||
|
run = plan_placement(db_sync, tmp_path)
|
||||||
|
src.unlink()
|
||||||
|
|
||||||
|
run = apply_run(db_sync, run)
|
||||||
|
|
||||||
|
assert run.refusals[0]["reason"] == "source missing"
|
||||||
|
db_sync.expire_all()
|
||||||
|
# The row still points at the missing file rather than at a file that
|
||||||
|
# was never created — a broken row is recoverable, a lying one is not.
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).path == str(src)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_plan_scopes_to_one_artist(db_sync, tmp_path):
|
||||||
|
"""Per-artist scope is what makes this incremental instead of one
|
||||||
|
irreversible sweep."""
|
||||||
|
from backend.app.services.library_layout import plan_placement
|
||||||
|
|
||||||
|
conto, _, _ = _staged(db_sync, tmp_path, "conto", "Conto", n=26)
|
||||||
|
_staged(db_sync, tmp_path, "maewix", "Maewix", name="y.png", n=27)
|
||||||
|
|
||||||
|
run = plan_placement(db_sync, tmp_path, artist_id=conto.id)
|
||||||
|
|
||||||
|
assert run.planned_count == 1
|
||||||
|
assert run.artist_id == conto.id
|
||||||
|
assert "Conto" in run.moves[0]["from"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_plan_skips_both_rows_when_two_want_one_destination(db_sync, tmp_path):
|
||||||
|
"""Which of two colliding rows 'wins' is not this sweep's call."""
|
||||||
|
from backend.app.services.library_layout import plan_placement
|
||||||
|
|
||||||
|
artist = _artist(db_sync, "Sticky", "sticky")
|
||||||
|
for stray, n in (("StickySpoodge", 28), ("Stickyspoodge", 29)):
|
||||||
|
p = tmp_path / stray / "dup.png"
|
||||||
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
p.write_bytes(b"pixels")
|
||||||
|
_image(db_sync, str(p), artist, n)
|
||||||
|
|
||||||
|
run = plan_placement(db_sync, tmp_path)
|
||||||
|
|
||||||
|
assert run.planned_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_thumbnails_do_not_move(db_sync, tmp_path):
|
||||||
|
"""Thumbs are sha-addressed (`thumbs/<xx>/<sha>.jpg`), not path-keyed, so
|
||||||
|
a placement move must not touch them. Pinned so nobody 'fixes' it."""
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
artist, rec, _ = _staged(db_sync, tmp_path, "conto", "Conto", n=30)
|
||||||
|
thumb = tmp_path / "thumbs" / "ab" / "abc.jpg"
|
||||||
|
thumb.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
thumb.write_bytes(b"thumb")
|
||||||
|
rec.thumbnail_path = str(thumb)
|
||||||
|
db_sync.flush()
|
||||||
|
|
||||||
|
apply_run(db_sync, plan_placement(db_sync, tmp_path))
|
||||||
|
|
||||||
|
db_sync.expire_all()
|
||||||
|
assert thumb.exists()
|
||||||
|
assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_apply_refuses_a_run_that_is_not_ready(db_sync, tmp_path):
|
||||||
|
from backend.app.services.library_layout import apply_run, plan_placement
|
||||||
|
|
||||||
|
_staged(db_sync, tmp_path, "conto", "Conto", n=31)
|
||||||
|
run = apply_run(db_sync, plan_placement(db_sync, tmp_path))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
apply_run(db_sync, run)
|
||||||
|
|||||||
Reference in New Issue
Block a user