feat(attachments): scan enumerates all; import_file maps 'attached' + member ML
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -26,11 +26,12 @@ IMAGES_ROOT = Path("/images")
|
|||||||
|
|
||||||
def _map_result_to_status(result):
|
def _map_result_to_status(result):
|
||||||
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
"""(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult.
|
||||||
'superseded' = the kept row's file/ML changed → complete + re-derive."""
|
'superseded' = the kept row's file/ML changed → complete + re-derive.
|
||||||
if result.status == "imported":
|
'attached' = a non-art file preserved → complete, no ML/thumb."""
|
||||||
return ("complete", True)
|
if result.status in ("imported", "superseded"):
|
||||||
if result.status == "superseded":
|
|
||||||
return ("complete", True)
|
return ("complete", True)
|
||||||
|
if result.status == "attached":
|
||||||
|
return ("complete", False)
|
||||||
if result.status == "skipped":
|
if result.status == "skipped":
|
||||||
return ("skipped", False)
|
return ("skipped", False)
|
||||||
return ("failed", False)
|
return ("failed", False)
|
||||||
@@ -82,6 +83,10 @@ def import_media_file(self, import_task_id: int) -> dict:
|
|||||||
task.result_image_id = result.image_id
|
task.result_image_id = result.image_id
|
||||||
counter_col_name = "imported"
|
counter_col_name = "imported"
|
||||||
counter_col = ImportBatch.imported
|
counter_col = ImportBatch.imported
|
||||||
|
elif result.status == "attached":
|
||||||
|
task.status = "complete"
|
||||||
|
counter_col_name = "attachments"
|
||||||
|
counter_col = ImportBatch.attachments
|
||||||
elif result.status == "skipped":
|
elif result.status == "skipped":
|
||||||
task.status = "skipped"
|
task.status = "skipped"
|
||||||
task.error = (
|
task.error = (
|
||||||
@@ -109,15 +114,16 @@ def import_media_file(self, import_task_id: int) -> dict:
|
|||||||
|
|
||||||
# Enqueue thumbnail + ML for newly imported AND superseded images
|
# Enqueue thumbnail + ML for newly imported AND superseded images
|
||||||
# (a superseded row has cleared ML + no thumbnail).
|
# (a superseded row has cleared ML + no thumbnail).
|
||||||
if (
|
if result.status in ("imported", "superseded"):
|
||||||
result.status in ("imported", "superseded")
|
|
||||||
and result.image_id is not None
|
|
||||||
):
|
|
||||||
from .ml import tag_and_embed
|
from .ml import tag_and_embed
|
||||||
from .thumbnail import generate_thumbnail
|
from .thumbnail import generate_thumbnail
|
||||||
|
|
||||||
generate_thumbnail.delay(result.image_id)
|
ids = list(result.member_image_ids)
|
||||||
tag_and_embed.delay(result.image_id)
|
if result.image_id is not None and result.image_id not in ids:
|
||||||
|
ids.append(result.image_id)
|
||||||
|
for img_id in ids:
|
||||||
|
generate_thumbnail.delay(img_id)
|
||||||
|
tag_and_embed.delay(img_id)
|
||||||
|
|
||||||
# If this was the last task in the batch, mark the batch complete.
|
# If this was the last task in the batch, mark the batch complete.
|
||||||
remaining = session.execute(
|
remaining = session.execute(
|
||||||
|
|||||||
@@ -11,7 +11,20 @@ from sqlalchemy.orm import sessionmaker
|
|||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..config import get_config
|
from ..config import get_config
|
||||||
from ..models import ImportBatch, ImportSettings, ImportTask
|
from ..models import ImportBatch, ImportSettings, ImportTask
|
||||||
from ..services.importer import is_supported
|
|
||||||
|
|
||||||
|
def _iter_import_files(import_root: Path):
|
||||||
|
"""Every regular file except sidecar .json, dotfiles, and .partial
|
||||||
|
temp files. The Importer dispatches by kind (media / archive /
|
||||||
|
other) — scan no longer filters to media (FC-2d-iii)."""
|
||||||
|
for entry in import_root.rglob("*"):
|
||||||
|
if not entry.is_file():
|
||||||
|
continue
|
||||||
|
if entry.name.startswith("."):
|
||||||
|
continue
|
||||||
|
if entry.suffix.lower() in (".json", ".partial"):
|
||||||
|
continue
|
||||||
|
yield entry
|
||||||
|
|
||||||
|
|
||||||
def _sync_session_factory():
|
def _sync_session_factory():
|
||||||
@@ -42,11 +55,7 @@ def scan_directory(self, triggered_by: str = "manual") -> int:
|
|||||||
|
|
||||||
# Walk and enumerate.
|
# Walk and enumerate.
|
||||||
files_seen = 0
|
files_seen = 0
|
||||||
for entry in import_root.rglob("*"):
|
for entry in _iter_import_files(import_root):
|
||||||
if not entry.is_file():
|
|
||||||
continue
|
|
||||||
if not is_supported(entry):
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
size = entry.stat().st_size
|
size = entry.stat().st_size
|
||||||
except OSError:
|
except OSError:
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""FC-2d-iii: import_file maps 'attached' + enqueues ML for members."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.app.services.importer import ImportResult
|
||||||
|
from backend.app.tasks.import_file import _map_result_to_status
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration # imports the celery app graph
|
||||||
|
|
||||||
|
|
||||||
|
def test_attached_maps_to_complete_no_requeue():
|
||||||
|
status, requeue = _map_result_to_status(
|
||||||
|
ImportResult(status="attached")
|
||||||
|
)
|
||||||
|
assert status == "complete"
|
||||||
|
assert requeue is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_imported_still_requeues():
|
||||||
|
status, requeue = _map_result_to_status(
|
||||||
|
ImportResult(status="imported", image_id=5)
|
||||||
|
)
|
||||||
|
assert status == "complete"
|
||||||
|
assert requeue is True
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""FC-2d-iii: scan no longer filters to media-only."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_includes_non_media(tmp_path):
|
||||||
|
from backend.app.tasks import scan as scan_mod
|
||||||
|
|
||||||
|
root = tmp_path / "imp"
|
||||||
|
(root / "Alice").mkdir(parents=True)
|
||||||
|
(root / "Alice" / "pic.jpg").write_bytes(b"x")
|
||||||
|
(root / "Alice" / "pack.zip").write_bytes(b"x")
|
||||||
|
(root / "Alice" / "doc.pdf").write_bytes(b"x")
|
||||||
|
(root / "Alice" / "meta.json").write_text("{}")
|
||||||
|
(root / "Alice" / ".hidden").write_text("x")
|
||||||
|
|
||||||
|
seen = sorted(p.name for p in scan_mod._iter_import_files(root))
|
||||||
|
assert seen == ["doc.pdf", "pack.zip", "pic.jpg"] # no json/dotfile
|
||||||
Reference in New Issue
Block a user