feat(deep-scan): IR-parity port — refreshed status + counter, re-queue completed paths in deep mode, honest UX — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -53,6 +53,7 @@ async def status():
|
||||
"imported": active.imported,
|
||||
"skipped": active.skipped,
|
||||
"failed": active.failed,
|
||||
"refreshed": active.refreshed,
|
||||
"started_at": active.started_at.isoformat(),
|
||||
}
|
||||
return jsonify(payload)
|
||||
|
||||
@@ -26,6 +26,10 @@ class ImportBatch(Base):
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||
# running | complete | cancelled
|
||||
|
||||
@@ -51,7 +51,14 @@ class SkipReason(StrEnum):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
||||
# 'imported' — new ImageRecord row created
|
||||
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
|
||||
# 'attached' — non-media saved as PostAttachment
|
||||
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
|
||||
# artist on an already-imported row (no new ImageRecord)
|
||||
# 'skipped' — no work done (true duplicate, too small, etc.)
|
||||
# 'failed' — pipeline error
|
||||
status: str
|
||||
image_id: int | None = None
|
||||
skip_reason: SkipReason | None = None
|
||||
error: str | None = None
|
||||
@@ -417,7 +424,18 @@ class Importer:
|
||||
) -> ImportResult:
|
||||
"""Deep scan: backfill phash/provenance/artist on an
|
||||
already-imported record. METADATA ONLY — never re-runs the pHash
|
||||
near-dup / supersede path. NULL-only, idempotent."""
|
||||
near-dup / supersede path. NULL-only on phash/artist, additive on
|
||||
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
|
||||
Idempotent: a second deep-scan over the same file finds nothing
|
||||
to refresh and is a no-op.
|
||||
|
||||
Returns status="refreshed" so the UI can surface the work done
|
||||
instead of the prior misleading "skipped/duplicate_hash" reading.
|
||||
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
|
||||
as a no-op skip during the original port and the UI showed deep
|
||||
scan as "completed with no changes" even when sidecar metadata
|
||||
actually got re-applied to N existing rows.
|
||||
"""
|
||||
if existing.phash is None and not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
@@ -436,10 +454,7 @@ class Importer:
|
||||
|
||||
self._apply_sidecar(existing, attribution_path, artist)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="deep: re-derived",
|
||||
)
|
||||
return ImportResult(status="refreshed", image_id=existing.id)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
|
||||
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
|
||||
def _map_result_to_status(result):
|
||||
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
||||
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb."""
|
||||
'attached' = a non-art file preserved → complete, no ML/thumb.
|
||||
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
|
||||
complete, no ML/thumb re-derive (file/pixels unchanged)."""
|
||||
if result.status in ("imported", "superseded"):
|
||||
return ("complete", True)
|
||||
if result.status == "attached":
|
||||
if result.status in ("attached", "refreshed"):
|
||||
return ("complete", False)
|
||||
if result.status == "skipped":
|
||||
return ("skipped", False)
|
||||
@@ -138,6 +140,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "imported"
|
||||
counter_col = ImportBatch.imported
|
||||
elif result.status == "refreshed":
|
||||
# Deep-scan rederive: existing row got phash/artist/sidecar
|
||||
# refreshed. Task is complete (no further work), but counted in
|
||||
# `refreshed` not `imported` so the UI can surface the actual
|
||||
# work done. operator-flagged 2026-05-25.
|
||||
task.status = "complete"
|
||||
task.result_image_id = result.image_id
|
||||
counter_col_name = "refreshed"
|
||||
counter_col = ImportBatch.refreshed
|
||||
elif result.status == "attached":
|
||||
task.status = "complete"
|
||||
counter_col_name = "attachments"
|
||||
|
||||
@@ -59,16 +59,25 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
session.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
# Skip-set: any source_path that already has a non-failed ImportTask
|
||||
# row. Re-running scan_directory must not re-enqueue files the
|
||||
# importer has already handled (or is currently handling); doing so
|
||||
# creates duplicate work and inflates the queue. Failed prior tasks
|
||||
# are eligible for retry.
|
||||
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
|
||||
#
|
||||
# quick: any non-failed prior ImportTask (active OR finished) is
|
||||
# skipped — quick scan only does new-file enqueue, so re-touching
|
||||
# already-imported files is wasted work.
|
||||
#
|
||||
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
|
||||
# are skipped. Completed and skipped tasks ARE re-queued because
|
||||
# deep scan exists precisely to re-touch already-imported files
|
||||
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
|
||||
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
|
||||
active_statuses = ["pending", "queued", "processing"]
|
||||
if mode == "deep":
|
||||
skip_statuses = active_statuses
|
||||
else:
|
||||
skip_statuses = active_statuses + ["complete", "skipped"]
|
||||
non_failed_existing = set(session.execute(
|
||||
select(ImportTask.source_path).where(
|
||||
ImportTask.status.in_(
|
||||
["pending", "queued", "processing", "complete", "skipped"]
|
||||
),
|
||||
ImportTask.status.in_(skip_statuses),
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user