fix(import): scan_directory skips paths with existing non-failed ImportTask
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -65,16 +65,34 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
session.flush()
|
session.flush()
|
||||||
batch_id = batch.id
|
batch_id = batch.id
|
||||||
|
|
||||||
|
# Skip-set: any source_path that already has a non-failed ImportTask
|
||||||
|
# row. Re-running scan_directory must not re-enqueue files the
|
||||||
|
# importer has already handled (or is currently handling); doing so
|
||||||
|
# creates duplicate work and inflates the queue. Failed prior tasks
|
||||||
|
# are eligible for retry.
|
||||||
|
non_failed_existing = set(session.execute(
|
||||||
|
select(ImportTask.source_path).where(
|
||||||
|
ImportTask.status.in_(
|
||||||
|
["pending", "queued", "processing", "complete", "skipped"]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
).scalars().all())
|
||||||
|
|
||||||
# Walk and enumerate.
|
# Walk and enumerate.
|
||||||
files_seen = 0
|
files_seen = 0
|
||||||
|
files_skipped_existing = 0
|
||||||
for entry in _iter_import_files(import_root):
|
for entry in _iter_import_files(import_root):
|
||||||
|
entry_str = str(entry)
|
||||||
|
if entry_str in non_failed_existing:
|
||||||
|
files_skipped_existing += 1
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
size = entry.stat().st_size
|
size = entry.stat().st_size
|
||||||
except OSError:
|
except OSError:
|
||||||
size = None
|
size = None
|
||||||
task = ImportTask(
|
task = ImportTask(
|
||||||
batch_id=batch_id,
|
batch_id=batch_id,
|
||||||
source_path=str(entry),
|
source_path=entry_str,
|
||||||
task_type="media",
|
task_type="media",
|
||||||
status="pending",
|
status="pending",
|
||||||
size_bytes=size,
|
size_bytes=size,
|
||||||
|
|||||||
Reference in New Issue
Block a user