335 lines
12 KiB
Python
335 lines
12 KiB
Python
"""Single-file import pipeline.
|
|
|
|
The Importer is the synchronous core of FC-2a's import path. A Celery task
|
|
(tasks/import_file.py) instantiates one per job and calls import_one().
|
|
|
|
The service is intentionally not async — Celery tasks are run in a
|
|
synchronous worker process, and the DB session is a sync session passed in
|
|
by the task. The Quart side uses an async session via the same models.
|
|
"""
|
|
|
|
import hashlib
|
|
import shutil
|
|
from dataclasses import dataclass
|
|
from enum import StrEnum
|
|
from pathlib import Path
|
|
|
|
from PIL import Image
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind
|
|
from ..models.tag import image_tag
|
|
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
|
from ..utils.phash import compute_phash, find_similar
|
|
from ..utils.slug import slugify
|
|
from .thumbnailer import Thumbnailer
|
|
|
|
|
|
class SkipReason(StrEnum):
|
|
too_small = "too_small"
|
|
too_transparent = "too_transparent"
|
|
single_color = "single_color"
|
|
duplicate_hash = "duplicate_hash"
|
|
duplicate_phash = "duplicate_phash"
|
|
invalid_image = "invalid_image"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ImportResult:
|
|
status: str # 'imported' | 'skipped' | 'failed' | 'superseded'
|
|
image_id: int | None = None
|
|
skip_reason: SkipReason | None = None
|
|
error: str | None = None
|
|
|
|
|
|
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
|
|
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"}
|
|
ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS
|
|
|
|
|
|
def is_supported(path: Path) -> bool:
|
|
return path.suffix.lower() in ALL_EXTS
|
|
|
|
|
|
def is_video(path: Path) -> bool:
|
|
return path.suffix.lower() in VIDEO_EXTS
|
|
|
|
|
|
def _mime_for(path: Path) -> str:
|
|
suffix = path.suffix.lower()
|
|
image_mimes = {
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".gif": "image/gif",
|
|
".webp": "image/webp",
|
|
".bmp": "image/bmp",
|
|
".tif": "image/tiff",
|
|
".tiff": "image/tiff",
|
|
}
|
|
video_mimes = {
|
|
".mp4": "video/mp4",
|
|
".mov": "video/quicktime",
|
|
".avi": "video/x-msvideo",
|
|
".mkv": "video/x-matroska",
|
|
".webm": "video/webm",
|
|
".m4v": "video/x-m4v",
|
|
".wmv": "video/x-ms-wmv",
|
|
".flv": "video/x-flv",
|
|
}
|
|
return image_mimes.get(suffix) or video_mimes.get(suffix) or "application/octet-stream"
|
|
|
|
|
|
def _sha256_of(path: Path, chunk_size: int = 1 << 20) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as fp:
|
|
while True:
|
|
block = fp.read(chunk_size)
|
|
if not block:
|
|
break
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
class Importer:
|
|
"""Imports one file at a time. Constructed per-task by the Celery task."""
|
|
|
|
def __init__(
|
|
self,
|
|
session: Session,
|
|
images_root: Path,
|
|
import_root: Path,
|
|
thumbnailer: Thumbnailer,
|
|
settings: ImportSettings,
|
|
):
|
|
self.session = session
|
|
self.images_root = images_root
|
|
self.import_root = import_root
|
|
self.thumbnailer = thumbnailer
|
|
self.settings = settings
|
|
|
|
def import_one(self, source: Path) -> ImportResult:
|
|
"""The full pipeline for a single file.
|
|
|
|
Returns an ImportResult; callers (the Celery task) are responsible for
|
|
translating that into ImportTask.status and ImageRecord linkage.
|
|
"""
|
|
if not is_supported(source):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=f"unsupported extension {source.suffix}",
|
|
)
|
|
|
|
# Compute file dimensions (images only) and apply filters.
|
|
width = height = None
|
|
has_alpha = False
|
|
if not is_video(source):
|
|
try:
|
|
with Image.open(source) as im:
|
|
im.verify()
|
|
with Image.open(source) as im:
|
|
width, height = im.size
|
|
has_alpha = im.mode in ("RGBA", "LA") or (
|
|
im.mode == "P" and "transparency" in im.info
|
|
)
|
|
except Exception as exc:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc),
|
|
)
|
|
|
|
if (self.settings.min_width and width < self.settings.min_width) or (
|
|
self.settings.min_height and height < self.settings.min_height
|
|
):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_small,
|
|
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
|
)
|
|
|
|
if self.settings.skip_transparent and has_alpha:
|
|
pct = self._transparency_pct(source)
|
|
if pct >= self.settings.transparency_threshold:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_transparent,
|
|
error=f"{pct:.2%} transparent",
|
|
)
|
|
|
|
# Hash dedup (exact).
|
|
sha = _sha256_of(source)
|
|
existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
|
|
existing = self.session.execute(existing_stmt).scalar_one_or_none()
|
|
if existing:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
|
image_id=existing.id, error="sha256 already present",
|
|
)
|
|
|
|
# Perceptual near-dup (images only; videos keep phash NULL).
|
|
phash = None
|
|
if not is_video(source):
|
|
with Image.open(source) as im:
|
|
phash = compute_phash(im)
|
|
if phash is not None:
|
|
cand_rows = self.session.execute(
|
|
select(
|
|
ImageRecord.phash,
|
|
ImageRecord.width,
|
|
ImageRecord.height,
|
|
ImageRecord.id,
|
|
).where(ImageRecord.phash.is_not(None))
|
|
).all()
|
|
candidates = [
|
|
(c.phash, c.width or 0, c.height or 0, c.id)
|
|
for c in cand_rows
|
|
]
|
|
rel, match_id = find_similar(
|
|
phash, width or 0, height or 0,
|
|
candidates, self.settings.phash_threshold,
|
|
)
|
|
if rel == "larger_exists":
|
|
return ImportResult(
|
|
status="skipped",
|
|
skip_reason=SkipReason.duplicate_phash,
|
|
image_id=match_id,
|
|
error="perceptual near-duplicate of larger existing image",
|
|
)
|
|
if rel == "smaller_exists":
|
|
target = self.session.get(ImageRecord, match_id)
|
|
self._supersede(target, source, sha, phash, width, height)
|
|
return ImportResult(
|
|
status="superseded", image_id=match_id
|
|
)
|
|
|
|
# Destination path.
|
|
subdir = derive_subdir(source, self.import_root)
|
|
dest_dir = self.images_root / subdir if subdir else self.images_root
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
dest_name = hash_suffixed_name(source.stem, sha, source.suffix)
|
|
dest = dest_dir / dest_name
|
|
|
|
# Atomic copy: write to .partial, rename. Avoids half-imported files on crash.
|
|
partial = dest.with_suffix(dest.suffix + ".partial")
|
|
shutil.copy2(source, partial)
|
|
partial.rename(dest)
|
|
|
|
record = ImageRecord(
|
|
path=str(dest),
|
|
sha256=sha,
|
|
phash=phash,
|
|
size_bytes=dest.stat().st_size,
|
|
mime=_mime_for(source),
|
|
width=width,
|
|
height=height,
|
|
origin="imported_filesystem",
|
|
integrity_status="unknown",
|
|
)
|
|
self.session.add(record)
|
|
self.session.flush()
|
|
|
|
# Folder→artist auto-tag.
|
|
artist_name = derive_top_level_artist(source, self.import_root)
|
|
if artist_name:
|
|
self._attach_artist_tag(record.id, artist_name)
|
|
|
|
# Thumbnail is queued separately by the calling task; the importer
|
|
# does not generate thumbnails inline so the import queue stays moving.
|
|
|
|
self.session.commit()
|
|
return ImportResult(status="imported", image_id=record.id)
|
|
|
|
def _attach_artist_tag(self, image_id: int, artist_name: str) -> None:
|
|
"""Idempotently creates Artist + artist tag, attaches to image."""
|
|
slug = slugify(artist_name)
|
|
artist = self.session.execute(
|
|
select(Artist).where(Artist.slug == slug)
|
|
).scalar_one_or_none()
|
|
if artist is None:
|
|
artist = Artist(name=artist_name, slug=slug, is_subscription=False)
|
|
self.session.add(artist)
|
|
self.session.flush()
|
|
|
|
tag = self.session.execute(
|
|
select(Tag).where(Tag.name == artist_name).where(Tag.kind == TagKind.artist)
|
|
).scalar_one_or_none()
|
|
if tag is None:
|
|
tag = Tag(name=artist_name, kind=TagKind.artist)
|
|
self.session.add(tag)
|
|
self.session.flush()
|
|
|
|
# Idempotent association; using a raw insert + on-conflict-do-nothing
|
|
# is cleaner here than fetching first, but for a sync session we just
|
|
# check existence.
|
|
already = self.session.execute(
|
|
select(image_tag.c.tag_id)
|
|
.where(image_tag.c.image_record_id == image_id)
|
|
.where(image_tag.c.tag_id == tag.id)
|
|
).scalar_one_or_none()
|
|
if already is None:
|
|
self.session.execute(
|
|
image_tag.insert().values(
|
|
image_record_id=image_id, tag_id=tag.id, source="auto",
|
|
)
|
|
)
|
|
|
|
def _supersede(
|
|
self, existing: ImageRecord, source: Path, sha: str,
|
|
phash: str, width: int | None, height: int | None,
|
|
) -> None:
|
|
"""Replace `existing`'s file with the larger `source`, keeping the
|
|
row id (so tags/series/curation stay attached). ML is cleared so
|
|
the import task re-derives it on the new pixels."""
|
|
subdir = derive_subdir(source, self.import_root)
|
|
dest_dir = self.images_root / subdir if subdir else self.images_root
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
dest = dest_dir / hash_suffixed_name(source.stem, sha, source.suffix)
|
|
partial = dest.with_suffix(dest.suffix + ".partial")
|
|
shutil.copy2(source, partial)
|
|
partial.rename(dest)
|
|
|
|
old_path = existing.path
|
|
old_thumb = existing.thumbnail_path
|
|
|
|
existing.path = str(dest)
|
|
existing.sha256 = sha
|
|
existing.phash = phash
|
|
existing.size_bytes = dest.stat().st_size
|
|
existing.mime = _mime_for(source)
|
|
existing.width = width
|
|
existing.height = height
|
|
existing.thumbnail_path = None
|
|
existing.integrity_status = "unknown"
|
|
existing.tagger_predictions = None
|
|
existing.tagger_model_version = None
|
|
existing.siglip_embedding = None
|
|
existing.siglip_model_version = None
|
|
existing.centroid_scores = None
|
|
# created_at intentionally preserved; updated_at auto-bumps.
|
|
self.session.flush()
|
|
self.session.commit()
|
|
|
|
for stale in (old_path, old_thumb):
|
|
if not stale:
|
|
continue
|
|
try:
|
|
p = Path(stale)
|
|
if p.exists():
|
|
p.unlink()
|
|
except Exception:
|
|
# Benign orphan; the DB swap already committed. Don't undo it.
|
|
pass
|
|
|
|
def _transparency_pct(self, source: Path) -> float:
|
|
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
|
with Image.open(source) as im:
|
|
if im.mode not in ("RGBA", "LA") and not (
|
|
im.mode == "P" and "transparency" in im.info
|
|
):
|
|
return 0.0
|
|
if im.mode != "RGBA":
|
|
im = im.convert("RGBA")
|
|
alpha = im.getchannel("A")
|
|
histogram = alpha.histogram()
|
|
transparent = histogram[0]
|
|
total = sum(histogram)
|
|
return transparent / total if total else 0.0
|