bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

feat(deep-scan): IR-parity port — refreshed status + counter, re-queue completed paths in deep mode, honest UX — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-05-25 22:16:26 -04:00
parent c06cbc0abe
commit e6ededbe8e
11 changed files with 251 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
alembic/versions/0019_import_batch_refreshed.py
+38
View File
@@ -0,0 +1,38 @@
"""import_batch.refreshed counter for deep-scan sidecar re-application
Revision ID: 0019
Revises: 0018
Create Date: 2026-05-25
Adds a `refreshed` counter to `import_batch`, mirroring the existing
`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now
re-applies sidecar metadata to already-imported files (the IR feature
that didn't make the FC port the first time); a "refreshed" outcome
increments this counter so the UI can surface "X new, Y refreshed"
instead of the misleading "Scan complete — no new files" message.
server_default=0 backfills existing rows in place — no UPDATE needed.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0019"
down_revision: Union[str, None] = "0018"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"import_batch",
sa.Column(
"refreshed", sa.Integer(),
nullable=False, server_default=sa.text("0"),
),
)
def downgrade() -> None:
op.drop_column("import_batch", "refreshed")
backend/app/api/import_admin.py
+1
View File
@@ -53,6 +53,7 @@ async def status():
"imported": active.imported,
"skipped": active.skipped,
"failed": active.failed,
"refreshed": active.refreshed,
"started_at": active.started_at.isoformat(),
}
return jsonify(payload)
backend/app/models/import_batch.py
+4
View File
@@ -26,6 +26,10 @@ class ImportBatch(Base):
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Deep-scan only: count of already-imported files whose sidecar metadata
# got re-applied this run (post/source/provenance upsert). Stays 0 on
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
# running | complete | cancelled
backend/app/services/importer.py
+21 -6
View File
@@ -51,7 +51,14 @@ class SkipReason(StrEnum):
@dataclass(frozen=True)
class ImportResult:
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
# 'imported' — new ImageRecord row created
# 'superseded' — existing ImageRecord row got the new file (larger) + sidecar
# 'attached' — non-media saved as PostAttachment
# 'refreshed' — deep scan re-applied sidecar / filled NULL phash / NULL
# artist on an already-imported row (no new ImageRecord)
# 'skipped' — no work done (true duplicate, too small, etc.)
# 'failed' — pipeline error
status: str
image_id: int | None = None
skip_reason: SkipReason | None = None
error: str | None = None
@@ -417,7 +424,18 @@ class Importer:
) -> ImportResult:
"""Deep scan: backfill phash/provenance/artist on an
already-imported record. METADATA ONLY — never re-runs the pHash
near-dup / supersede path. NULL-only, idempotent."""
near-dup / supersede path. NULL-only on phash/artist, additive on
sidecar Post/Source/ImageProvenance (via _apply_sidecar).
Idempotent: a second deep-scan over the same file finds nothing
to refresh and is a no-op.
Returns status="refreshed" so the UI can surface the work done
instead of the prior misleading "skipped/duplicate_hash" reading.
Operator-flagged 2026-05-25 — IR has had this; FC inherited it
as a no-op skip during the original port and the UI showed deep
scan as "completed with no changes" even when sidecar metadata
actually got re-applied to N existing rows.
"""
if existing.phash is None and not is_video(source):
try:
with Image.open(source) as im:
@@ -436,10 +454,7 @@ class Importer:
self._apply_sidecar(existing, attribution_path, artist)
self.session.commit()
return ImportResult(
status="skipped", skip_reason=SkipReason.duplicate_hash,
image_id=existing.id, error="deep: re-derived",
)
return ImportResult(status="refreshed", image_id=existing.id)
def attach_in_place(
self,
backend/app/tasks/import_file.py
+13 -2
View File
@@ -29,10 +29,12 @@ IMAGES_ROOT = Path("/images")
def _map_result_to_status(result):
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
'superseded' = the kept row's file/ML changed → complete + re-derive.
'attached' = a non-art file preserved → complete, no ML/thumb."""
'attached' = a non-art file preserved → complete, no ML/thumb.
'refreshed' = deep scan refreshed sidecar/phash on an existing row →
complete, no ML/thumb re-derive (file/pixels unchanged)."""
if result.status in ("imported", "superseded"):
return ("complete", True)
if result.status == "attached":
if result.status in ("attached", "refreshed"):
return ("complete", False)
if result.status == "skipped":
return ("skipped", False)
@@ -138,6 +140,15 @@ def _do_import(session, task, import_task_id: int) -> dict:
task.result_image_id = result.image_id
counter_col_name = "imported"
counter_col = ImportBatch.imported
elif result.status == "refreshed":
# Deep-scan rederive: existing row got phash/artist/sidecar
# refreshed. Task is complete (no further work), but counted in
# `refreshed` not `imported` so the UI can surface the actual
# work done. operator-flagged 2026-05-25.
task.status = "complete"
task.result_image_id = result.image_id
counter_col_name = "refreshed"
counter_col = ImportBatch.refreshed
elif result.status == "attached":
task.status = "complete"
counter_col_name = "attachments"
backend/app/tasks/scan.py
+17 -8
View File
@@ -59,16 +59,25 @@ def scan_directory(self, triggered_by: str = "manual",
session.flush()
batch_id = batch.id
# Skip-set: any source_path that already has a non-failed ImportTask
# row. Re-running scan_directory must not re-enqueue files the
# importer has already handled (or is currently handling); doing so
# creates duplicate work and inflates the queue. Failed prior tasks
# are eligible for retry.
# Skip-set behavior splits by mode (operator-flagged 2026-05-25):
#
# quick: any non-failed prior ImportTask (active OR finished) is
# skipped — quick scan only does new-file enqueue, so re-touching
# already-imported files is wasted work.
#
# deep: ONLY currently-in-flight tasks (pending/queued/processing)
# are skipped. Completed and skipped tasks ARE re-queued because
# deep scan exists precisely to re-touch already-imported files
# (refresh sidecar metadata, fill NULL phash, fill NULL artist
# via Importer._deep_rederive). Matches IR's deep-scan behavior.
active_statuses = ["pending", "queued", "processing"]
if mode == "deep":
skip_statuses = active_statuses
else:
skip_statuses = active_statuses + ["complete", "skipped"]
non_failed_existing = set(session.execute(
select(ImportTask.source_path).where(
ImportTask.status.in_(
["pending", "queued", "processing", "complete", "skipped"]
),
ImportTask.status.in_(skip_statuses),
)
).scalars().all())
frontend/src/components/settings/ImportTriggerPanel.vue
+9 -5
View File
@@ -10,6 +10,9 @@
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
{{ store.activeBatch.source_path || '/import' }} —
imported {{ store.activeBatch.imported }},
<template v-if="store.activeBatch.scan_mode === 'deep'">
refreshed {{ store.activeBatch.refreshed || 0 }},
</template>
skipped {{ store.activeBatch.skipped }},
failed {{ store.activeBatch.failed }} /
{{ store.activeBatch.total_files }} files
@@ -26,11 +29,12 @@
<p class="text-body-2 mb-3">
<span v-if="!store.activeBatch">
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
new files only (skips paths already on a non-failed ImportTask).
<strong>Deep scan</strong> additionally chains a phash backfill
across the existing library — use after bulk-imports to catch
near-duplicates that slipped through. Both modes route non-media
+ sidecar pairs through PostAttachment capture.
new files only.
<strong>Deep scan</strong> additionally re-walks already-imported
files so updated sidecar metadata (post title/date/attribution) and
previously-NULL phashes / artist links get refreshed. Use after
bulk-downloading fresh sidecars for existing content. Both modes
route non-media + sidecar pairs through PostAttachment capture.
</span>
<span v-else>
An active batch is in progress. Wait for it to finish, or click
frontend/src/stores/import.js
+27 -6
View File
@@ -66,21 +66,42 @@ export const useImportStore = defineStore('import', () => {
// batch flashes 'running' for <100ms then 'complete' before the
// first refreshStatus() lands; UI never sees the active state).
const label = mode === 'deep'
? 'Deep scan triggered (pHash backfill chained)'
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
window.__fcToast?.({ text: label, type: 'success' })
await refreshStatus()
// Re-poll twice over ~5s to catch quick-finalize transitions and
// surface a result toast either way. Skip the "no new files" hint
// for 'verify' since it doesn't walk the import_root.
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
// Operator-flagged 2026-05-25: the prior "no new files" message was
// misleading because deep scan IS doing work (refresh) even when
// there are no new files to import. Surface the real workload count
// (imported + refreshed + queued) instead. For quick scan + zero
// queued work, fall back to "up to date" instead of the old
// implementation-detail-leaking message.
setTimeout(async () => {
await refreshStatus()
await loadTasks(true)
if (!activeBatch.value && mode !== 'verify') {
if (activeBatch.value || mode === 'verify') return
// Batch finalized quickly; figure out what actually happened.
// The task list was just refreshed; the freshest row(s) carry
// the batch outcome.
const batchId = tasks.value[0]?.batch_id
const sameBatch = batchId
? tasks.value.filter(t => t.batch_id === batchId)
: []
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
if (mode === 'deep' && sameBatch.length > 0) {
window.__fcToast?.({
text: 'Scan complete — no new files (everything already on an ImportTask row)',
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
type: 'info',
})
} else if (mode === 'quick' && sameBatch.length > 0) {
window.__fcToast?.({
text: `Quick scan finished — ${newImported} new file(s) queued`,
type: 'info',
})
} else {
window.__fcToast?.({ text: 'Library is up to date', type: 'info' })
}
}, 2000)
} catch (e) {
tests/test_importer_deep.py
+6 -2
View File
@@ -64,9 +64,13 @@ def test_deep_rederives_phash_and_provenance(db_sync, import_layout):
deep = _mk(db_sync, import_layout, deep=True)
r2 = deep.import_one(src)
assert r2.status == "skipped"
assert "deep" in (r2.error or "")
# Outcome flipped from "skipped+duplicate_hash" to "refreshed" 2026-05-25
# so the UI can surface deep scan's actual work instead of showing it as
# a no-op. See ImportResult.status comment + _deep_rederive docstring.
assert r2.status == "refreshed"
assert r2.image_id == rec.id
assert r2.skip_reason is None
assert r2.error is None
db_sync.expire_all()
rec2 = db_sync.get(ImageRecord, rec.id)
tests/test_phash_dedup.py
+6
View File
@@ -241,3 +241,9 @@ def test_import_task_maps_superseded_to_complete_and_requeues():
assert _map_result_to_status(
ImportResult(status="failed", error="boom")
) == ("failed", False)
# Refreshed (deep scan): complete + no ML/thumb re-derive (pixels
# unchanged). Added 2026-05-25 alongside ImportBatch.refreshed
# counter so deep scan reports "X refreshed" instead of "no work".
assert _map_result_to_status(
ImportResult(status="refreshed", image_id=5)
) == ("complete", False)
tests/test_scan_deep_requeue.py
+109
View File
@@ -0,0 +1,109 @@
"""Deep scan re-queues already-completed ImportTasks.
Operator-flagged 2026-05-25: deep scan used to skip everything that
already had a non-failed ImportTask row, making a deep re-scan a no-op
when no new files were added. That defeated the entire point of deep
scan (re-apply sidecar metadata to existing rows). The skip-set now
splits by mode — quick keeps the old "any non-failed" semantics; deep
skips ONLY actively-in-flight statuses.
"""
import pytest
from PIL import Image
from sqlalchemy import func, select
from backend.app.models import ImportBatch, ImportSettings, ImportTask
from backend.app.tasks.scan import scan_directory
pytestmark = pytest.mark.integration
def _img(path):
path.parent.mkdir(parents=True, exist_ok=True)
Image.new("RGB", (40, 40), (10, 200, 80)).save(path, "JPEG")
def test_deep_scan_requeues_completed_task(db_sync, tmp_path, monkeypatch):
"""Quick scan then deep scan of the same /import: the file completed
in the first run should be re-enqueued by the deep run (different
ImportTask id, same source_path)."""
import_root = tmp_path / "import"
settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
settings.import_scan_path = str(import_root)
db_sync.commit()
src = import_root / "Mae" / "p.jpg"
_img(src)
from backend.app import celery_app
celery_app.celery.conf.task_always_eager = False # explicit
try:
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
first_task = db_sync.execute(
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
).scalar_one()
# Simulate the worker having finished it.
first_task.status = "complete"
db_sync.commit()
# Now deep scan: the SAME file should get a NEW ImportTask row.
second_batch_id = scan_directory.run(triggered_by="manual", mode="deep")
assert second_batch_id != first_batch_id
new_tasks_in_second_batch = db_sync.execute(
select(func.count())
.select_from(ImportTask)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert new_tasks_in_second_batch == 1, (
"deep scan did not re-queue the completed file"
)
# And the second batch's task should be for the same source_path
# as the first (proves it's a re-queue, not a different file).
sp = db_sync.execute(
select(ImportTask.source_path)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert sp == str(src)
finally:
celery_app.celery.conf.task_always_eager = False
def test_quick_scan_does_not_requeue_completed_task(db_sync, tmp_path):
"""The flip side: quick scan still keeps the old skip semantics —
a file with a completed ImportTask row from a prior batch is NOT
re-enqueued on a fresh quick scan."""
import_root = tmp_path / "import"
settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
settings.import_scan_path = str(import_root)
db_sync.commit()
src = import_root / "Mae" / "q.jpg"
_img(src)
first_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
first_task = db_sync.execute(
select(ImportTask).where(ImportTask.batch_id == first_batch_id)
).scalar_one()
first_task.status = "complete"
db_sync.commit()
second_batch_id = scan_directory.run(triggered_by="manual", mode="quick")
second_batch_count = db_sync.execute(
select(func.count())
.select_from(ImportTask)
.where(ImportTask.batch_id == second_batch_id)
).scalar_one()
assert second_batch_count == 0, (
"quick scan re-queued an already-completed task — should have skipped"
)
# And the second batch should self-finalize (files_seen=0).
second_batch = db_sync.get(ImportBatch, second_batch_id)
assert second_batch.status == "complete"