831 lines
32 KiB
Python
831 lines
32 KiB
Python
"""Single-file import pipeline.
|
|
|
|
The Importer is the synchronous core of FC-2a's import path. A Celery task
|
|
(tasks/import_file.py) instantiates one per job and calls import_one().
|
|
|
|
The service is intentionally not async — Celery tasks are run in a
|
|
synchronous worker process, and the DB session is a sync session passed in
|
|
by the task. The Quart side uses an async session via the same models.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import shutil
|
|
from dataclasses import dataclass, field
|
|
from enum import StrEnum
|
|
from pathlib import Path
|
|
|
|
from PIL import Image
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..models import (
|
|
Artist,
|
|
ImageProvenance,
|
|
ImageRecord,
|
|
ImportSettings,
|
|
Post,
|
|
PostAttachment,
|
|
Source,
|
|
)
|
|
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
|
from ..utils.phash import compute_phash, find_similar
|
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
|
from ..utils.slug import slugify
|
|
from .archive_extractor import extract_archive, is_archive
|
|
from .attachment_store import AttachmentStore
|
|
from .thumbnailer import Thumbnailer
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class SkipReason(StrEnum):
|
|
too_small = "too_small"
|
|
too_transparent = "too_transparent"
|
|
single_color = "single_color"
|
|
duplicate_hash = "duplicate_hash"
|
|
duplicate_phash = "duplicate_phash"
|
|
invalid_image = "invalid_image"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ImportResult:
|
|
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
|
image_id: int | None = None
|
|
skip_reason: SkipReason | None = None
|
|
error: str | None = None
|
|
member_image_ids: list[int] = field(default_factory=list)
|
|
|
|
|
|
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
|
|
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"}
|
|
ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS
|
|
|
|
|
|
def is_supported(path: Path) -> bool:
|
|
return path.suffix.lower() in ALL_EXTS
|
|
|
|
|
|
def is_video(path: Path) -> bool:
|
|
return path.suffix.lower() in VIDEO_EXTS
|
|
|
|
|
|
def _safe_ext(path: Path) -> str:
|
|
"""Conservatively extract a file extension for PostAttachment.ext
|
|
(varchar(32)).
|
|
|
|
gallery-dl produces some filenames with URL-encoded query-string
|
|
artifacts embedded into the basename (e.g.
|
|
`79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
|
|
`Path.suffix` finds the LAST dot and returns everything after, which
|
|
in those cases yields a 50+ char "extension" of mostly base64-ish
|
|
junk. That blows the column. Operator-flagged 2026-05-25.
|
|
|
|
Real extensions are short and alphanumeric. We accept anything ≤ 16
|
|
chars where every post-dot character is alphanumeric; anything else
|
|
means the input wasn't a real extension and we return the empty
|
|
string. ext is nullable-ish (empty string still satisfies NOT NULL)
|
|
and consumers should treat "" as "no known extension".
|
|
"""
|
|
suffix = path.suffix.lower()
|
|
if not suffix or len(suffix) > 16:
|
|
return ""
|
|
if not all(c.isalnum() for c in suffix[1:]):
|
|
return ""
|
|
return suffix
|
|
|
|
|
|
def _mime_for(path: Path) -> str:
|
|
suffix = path.suffix.lower()
|
|
image_mimes = {
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".gif": "image/gif",
|
|
".webp": "image/webp",
|
|
".bmp": "image/bmp",
|
|
".tif": "image/tiff",
|
|
".tiff": "image/tiff",
|
|
}
|
|
video_mimes = {
|
|
".mp4": "video/mp4",
|
|
".mov": "video/quicktime",
|
|
".avi": "video/x-msvideo",
|
|
".mkv": "video/x-matroska",
|
|
".webm": "video/webm",
|
|
".m4v": "video/x-m4v",
|
|
".wmv": "video/x-ms-wmv",
|
|
".flv": "video/x-flv",
|
|
}
|
|
return image_mimes.get(suffix) or video_mimes.get(suffix) or "application/octet-stream"
|
|
|
|
|
|
def _sha256_of(path: Path, chunk_size: int = 1 << 20) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as fp:
|
|
while True:
|
|
block = fp.read(chunk_size)
|
|
if not block:
|
|
break
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
class Importer:
|
|
"""Imports one file at a time. Constructed per-task by the Celery task."""
|
|
|
|
def __init__(
|
|
self,
|
|
session: Session,
|
|
images_root: Path,
|
|
import_root: Path,
|
|
thumbnailer: Thumbnailer,
|
|
settings: ImportSettings,
|
|
deep: bool = False,
|
|
):
|
|
self.session = session
|
|
self.images_root = images_root
|
|
self.import_root = import_root
|
|
self.thumbnailer = thumbnailer
|
|
self.settings = settings
|
|
self.deep = deep
|
|
self.attachments = AttachmentStore(images_root)
|
|
|
|
def import_one(self, source: Path) -> ImportResult:
|
|
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
|
media members (one Post via the archive-adjacent sidecar) and
|
|
preserve the archive itself. Other non-media → PostAttachment.
|
|
Nothing is skipped for being non-media (FC-2d-iii)."""
|
|
if source.suffix.lower() == ".json":
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error="sidecar json is metadata, not content",
|
|
)
|
|
if is_archive(source):
|
|
return self._import_archive(source)
|
|
if not is_supported(source):
|
|
return self._capture_attachment(source)
|
|
return self._import_media(source, source)
|
|
|
|
def _resolve_artist(self, attribution_path: Path) -> Artist | None:
|
|
name = derive_top_level_artist(attribution_path, self.import_root)
|
|
return self._upsert_artist(name) if name else None
|
|
|
|
def _post_for_sidecar(
|
|
self, source: Path, artist: Artist | None
|
|
) -> Post | None:
|
|
"""If a sidecar sits next to `source`, ensure its Source+Post
|
|
exist (idempotent) and return the Post — so attachments can link
|
|
to the same Post the per-member _apply_sidecar will reuse."""
|
|
sc = find_sidecar(source)
|
|
if sc is None or artist is None:
|
|
return None
|
|
try:
|
|
data = json.loads(sc.read_text("utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError("sidecar JSON is not an object")
|
|
except Exception as exc:
|
|
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
|
return None
|
|
sd = parse_sidecar(data)
|
|
platform = sd.platform or "unknown"
|
|
url = sd.post_url or f"sidecar:{platform}"
|
|
src = self.session.execute(
|
|
select(Source).where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == platform,
|
|
Source.url == url,
|
|
)
|
|
).scalar_one_or_none()
|
|
if src is None:
|
|
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
self.session.add(src)
|
|
self.session.flush()
|
|
epid = sd.external_post_id or sc.stem
|
|
post = self.session.execute(
|
|
select(Post).where(
|
|
Post.source_id == src.id,
|
|
Post.external_post_id == epid,
|
|
)
|
|
).scalar_one_or_none()
|
|
if post is None:
|
|
post = Post(source_id=src.id, external_post_id=epid)
|
|
self.session.add(post)
|
|
self.session.flush()
|
|
return post
|
|
|
|
def _capture_attachment(
|
|
self, source: Path, *, post: Post | None = None,
|
|
artist: Artist | None = None, resolved: bool = False,
|
|
) -> ImportResult:
|
|
if not resolved:
|
|
artist = self._resolve_artist(source)
|
|
post = self._post_for_sidecar(source, artist)
|
|
sha = _sha256_of(source)
|
|
existing = self.session.execute(
|
|
select(PostAttachment).where(PostAttachment.sha256 == sha)
|
|
).scalar_one_or_none()
|
|
if existing is None:
|
|
stored = self.attachments.store(source, sha)
|
|
self.session.add(PostAttachment(
|
|
post_id=post.id if post else None,
|
|
artist_id=artist.id if artist else None,
|
|
sha256=sha,
|
|
path=stored,
|
|
original_filename=source.name,
|
|
ext=_safe_ext(source),
|
|
mime=_mime_for(source),
|
|
size_bytes=source.stat().st_size,
|
|
))
|
|
self.session.flush()
|
|
self.session.commit()
|
|
return ImportResult(status="attached")
|
|
|
|
def _import_archive(self, source: Path) -> ImportResult:
|
|
artist = self._resolve_artist(source)
|
|
post = self._post_for_sidecar(source, artist)
|
|
member_ids: list[int] = []
|
|
with extract_archive(source) as members:
|
|
for _name, member_path in members:
|
|
if not is_supported(member_path):
|
|
continue # non-media preserved via the stored archive
|
|
res = self._import_media(member_path, source)
|
|
if res.status in ("imported", "superseded") and res.image_id:
|
|
member_ids.append(res.image_id)
|
|
# Preserve the archive itself (links to the same Post/Artist).
|
|
self._capture_attachment(
|
|
source, post=post, artist=artist, resolved=True
|
|
)
|
|
if member_ids:
|
|
return ImportResult(
|
|
status="imported", image_id=member_ids[0],
|
|
member_image_ids=member_ids,
|
|
)
|
|
return ImportResult(status="attached")
|
|
|
|
def _import_media(
|
|
self, source: Path, attribution_path: Path
|
|
) -> ImportResult:
|
|
"""The media import pipeline (filters, dedup, copy, provenance).
|
|
|
|
`attribution_path` anchors artist/subdir/sidecar derivation: for
|
|
a normal file it equals `source`; for an archive member it is the
|
|
archive's real path (the member lives in a tempdir).
|
|
"""
|
|
if not is_supported(source):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=f"unsupported extension {source.suffix}",
|
|
)
|
|
|
|
# Compute file dimensions (images only) and apply filters.
|
|
width = height = None
|
|
has_alpha = False
|
|
if not is_video(source):
|
|
try:
|
|
with Image.open(source) as im:
|
|
im.verify()
|
|
with Image.open(source) as im:
|
|
width, height = im.size
|
|
has_alpha = im.mode in ("RGBA", "LA") or (
|
|
im.mode == "P" and "transparency" in im.info
|
|
)
|
|
except Exception as exc:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc),
|
|
)
|
|
|
|
if (self.settings.min_width and width < self.settings.min_width) or (
|
|
self.settings.min_height and height < self.settings.min_height
|
|
):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_small,
|
|
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
|
)
|
|
|
|
if self.settings.skip_transparent and has_alpha:
|
|
try:
|
|
pct = self._transparency_pct(source)
|
|
except OSError as exc:
|
|
# PIL.verify() at line 263 only validates header structure;
|
|
# truncated/corrupt pixel data only surfaces when load()
|
|
# actually decodes (here via getchannel('A')). Convert to
|
|
# invalid_image skip so the Celery autoretry loop doesn't
|
|
# bounce the same broken file forever.
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=f"PIL load failed during transparency check: {exc}",
|
|
)
|
|
if pct >= self.settings.transparency_threshold:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_transparent,
|
|
error=f"{pct:.2%} transparent",
|
|
)
|
|
|
|
# Hash dedup (exact).
|
|
sha = _sha256_of(source)
|
|
existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
|
|
existing = self.session.execute(existing_stmt).scalar_one_or_none()
|
|
if existing:
|
|
if not self.deep:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
|
image_id=existing.id, error="sha256 already present",
|
|
)
|
|
return self._deep_rederive(existing, source, attribution_path)
|
|
|
|
# Perceptual near-dup (images only; videos keep phash NULL).
|
|
phash = None
|
|
if not is_video(source):
|
|
try:
|
|
with Image.open(source) as im:
|
|
phash = compute_phash(im)
|
|
except OSError as exc:
|
|
# Same rationale as the transparency-check guard above:
|
|
# broken-pixel-data files pass verify() but blow up here.
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=f"PIL load failed during phash compute: {exc}",
|
|
)
|
|
if phash is not None:
|
|
cand_rows = self.session.execute(
|
|
select(
|
|
ImageRecord.phash,
|
|
ImageRecord.width,
|
|
ImageRecord.height,
|
|
ImageRecord.id,
|
|
).where(ImageRecord.phash.is_not(None))
|
|
).all()
|
|
candidates = [
|
|
(c.phash, c.width or 0, c.height or 0, c.id)
|
|
for c in cand_rows
|
|
]
|
|
rel, match_id = find_similar(
|
|
phash, width or 0, height or 0,
|
|
candidates, self.settings.phash_threshold,
|
|
)
|
|
if rel == "larger_exists":
|
|
return ImportResult(
|
|
status="skipped",
|
|
skip_reason=SkipReason.duplicate_phash,
|
|
image_id=match_id,
|
|
error="perceptual near-duplicate of larger existing image",
|
|
)
|
|
if rel == "smaller_exists":
|
|
target = self.session.get(ImageRecord, match_id)
|
|
self._supersede(target, source, sha, phash, width, height)
|
|
return ImportResult(
|
|
status="superseded", image_id=match_id
|
|
)
|
|
|
|
dest = self._copy_to_library(source, sha, attribution_path)
|
|
|
|
record = ImageRecord(
|
|
path=str(dest),
|
|
sha256=sha,
|
|
phash=phash,
|
|
size_bytes=dest.stat().st_size,
|
|
mime=_mime_for(source),
|
|
width=width,
|
|
height=height,
|
|
origin="imported_filesystem",
|
|
integrity_status="unknown",
|
|
)
|
|
self.session.add(record)
|
|
self.session.flush()
|
|
|
|
# Folder→artist (anchored to attribution_path).
|
|
artist = None
|
|
artist_name = derive_top_level_artist(
|
|
attribution_path, self.import_root
|
|
)
|
|
if artist_name:
|
|
artist = self._attach_artist(record, artist_name)
|
|
|
|
# Sidecar provenance (best-effort; never fails the import).
|
|
self._apply_sidecar(record, attribution_path, artist)
|
|
|
|
# Thumbnail is queued separately by the calling task; the importer
|
|
# does not generate thumbnails inline so the import queue stays moving.
|
|
|
|
self.session.commit()
|
|
return ImportResult(status="imported", image_id=record.id)
|
|
|
|
def _deep_rederive(
|
|
self, existing: ImageRecord, source: Path, attribution_path: Path
|
|
) -> ImportResult:
|
|
"""Deep scan: backfill phash/provenance/artist on an
|
|
already-imported record. METADATA ONLY — never re-runs the pHash
|
|
near-dup / supersede path. NULL-only, idempotent."""
|
|
if existing.phash is None and not is_video(source):
|
|
try:
|
|
with Image.open(source) as im:
|
|
ph = compute_phash(im)
|
|
if ph is not None:
|
|
existing.phash = ph
|
|
except Exception as exc:
|
|
log.warning("deep rephash failed for %s: %s", source, exc)
|
|
|
|
artist = None
|
|
name = derive_top_level_artist(attribution_path, self.import_root)
|
|
if name:
|
|
artist = self._upsert_artist(name)
|
|
if existing.artist_id is None:
|
|
existing.artist_id = artist.id
|
|
|
|
self._apply_sidecar(existing, attribution_path, artist)
|
|
self.session.commit()
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
|
image_id=existing.id, error="deep: re-derived",
|
|
)
|
|
|
|
def attach_in_place(
|
|
self,
|
|
path: Path,
|
|
*,
|
|
sidecar_path: Path | None = None,
|
|
artist: Artist | None = None,
|
|
source: Source | None = None,
|
|
) -> ImportResult:
|
|
"""Attach a file ALREADY at its final library location (FC-3c).
|
|
|
|
Mirrors _import_media but skips the copy step — the file is
|
|
where it'll permanently live. Caller (DownloadService) already
|
|
has artist/source context from the subscription Source row; pass
|
|
them through. The sidecar JSON gallery-dl emits next to each
|
|
downloaded file is read by `_apply_sidecar` via `find_sidecar`.
|
|
|
|
Caller's responsibilities after this returns:
|
|
- duplicate_hash / duplicate_phash skip → delete the on-disk file
|
|
- superseded → file stays where it is (now canonical)
|
|
- imported → file stays where it is
|
|
- failed → file untouched; caller decides
|
|
"""
|
|
if not is_supported(path):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=f"unsupported extension {path.suffix}",
|
|
)
|
|
|
|
# Format / dimension / transparency filters (mirror _import_media).
|
|
width = height = None
|
|
has_alpha = False
|
|
if not is_video(path):
|
|
try:
|
|
with Image.open(path) as im:
|
|
im.verify()
|
|
with Image.open(path) as im:
|
|
width, height = im.size
|
|
has_alpha = im.mode in ("RGBA", "LA") or (
|
|
im.mode == "P" and "transparency" in im.info
|
|
)
|
|
except Exception as exc:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
|
error=str(exc),
|
|
)
|
|
|
|
if (self.settings.min_width and width < self.settings.min_width) or (
|
|
self.settings.min_height and height < self.settings.min_height
|
|
):
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_small,
|
|
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
|
)
|
|
|
|
if self.settings.skip_transparent and has_alpha:
|
|
pct = self._transparency_pct(path)
|
|
if pct >= self.settings.transparency_threshold:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.too_transparent,
|
|
error=f"{pct:.2%} transparent",
|
|
)
|
|
|
|
# Hash dedup
|
|
sha = _sha256_of(path)
|
|
existing = self.session.execute(
|
|
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
|
).scalar_one_or_none()
|
|
if existing:
|
|
return ImportResult(
|
|
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
|
image_id=existing.id, error="sha256 already present",
|
|
)
|
|
|
|
# phash dedup + supersede
|
|
phash = None
|
|
if not is_video(path):
|
|
try:
|
|
with Image.open(path) as im:
|
|
phash = compute_phash(im)
|
|
except Exception:
|
|
phash = None
|
|
if phash is not None:
|
|
cand_rows = self.session.execute(
|
|
select(
|
|
ImageRecord.phash,
|
|
ImageRecord.width,
|
|
ImageRecord.height,
|
|
ImageRecord.id,
|
|
).where(ImageRecord.phash.is_not(None))
|
|
).all()
|
|
candidates = [
|
|
(c.phash, c.width or 0, c.height or 0, c.id)
|
|
for c in cand_rows
|
|
]
|
|
rel, match_id = find_similar(
|
|
phash, width or 0, height or 0,
|
|
candidates, self.settings.phash_threshold,
|
|
)
|
|
if rel == "larger_exists":
|
|
return ImportResult(
|
|
status="skipped",
|
|
skip_reason=SkipReason.duplicate_phash,
|
|
image_id=match_id,
|
|
error="perceptual near-duplicate of larger existing image",
|
|
)
|
|
if rel == "smaller_exists":
|
|
target = self.session.get(ImageRecord, match_id)
|
|
self._supersede(
|
|
target, path, sha, phash, width, height, new_path=path
|
|
)
|
|
return ImportResult(status="superseded", image_id=match_id)
|
|
|
|
# Create record — the path IS where the file lives.
|
|
record = ImageRecord(
|
|
path=str(path),
|
|
sha256=sha,
|
|
phash=phash,
|
|
size_bytes=path.stat().st_size,
|
|
mime=_mime_for(path),
|
|
width=width,
|
|
height=height,
|
|
origin="downloaded",
|
|
integrity_status="unknown",
|
|
)
|
|
if artist is not None:
|
|
record.artist_id = artist.id
|
|
self.session.add(record)
|
|
self.session.flush()
|
|
|
|
# Sidecar provenance (best-effort). When `source` is passed, link
|
|
# the post to that subscription Source instead of creating a new
|
|
# per-post Source row.
|
|
self._apply_sidecar(record, path, artist, explicit_source=source)
|
|
|
|
self.session.commit()
|
|
return ImportResult(status="imported", image_id=record.id)
|
|
|
|
def _upsert_artist(self, name: str) -> Artist:
|
|
slug = slugify(name)
|
|
artist = self.session.execute(
|
|
select(Artist).where(Artist.slug == slug)
|
|
).scalar_one_or_none()
|
|
if artist is None:
|
|
artist = Artist(name=name, slug=slug, is_subscription=False)
|
|
self.session.add(artist)
|
|
self.session.flush()
|
|
return artist
|
|
|
|
def _attach_artist(self, record: ImageRecord, artist_name: str) -> Artist:
|
|
"""Upsert the Artist and set it as the image's canonical artist.
|
|
Artist-kind tags were retired in FC-2d-vii-c — the Artist row is
|
|
the single source of truth. Returns the Artist (sidecar
|
|
provenance attaches its Source to it)."""
|
|
artist = self._upsert_artist(artist_name)
|
|
if record.artist_id is None:
|
|
record.artist_id = artist.id
|
|
self.session.flush()
|
|
return artist
|
|
|
|
@staticmethod
|
|
def _sidecar_artist_name(data: dict) -> str | None:
|
|
for k in ("artist", "author_name"):
|
|
v = data.get(k)
|
|
if isinstance(v, str) and v.strip():
|
|
return v.strip()
|
|
for k in ("user", "creator"):
|
|
v = data.get(k)
|
|
if isinstance(v, str) and v.strip():
|
|
return v.strip()
|
|
if isinstance(v, dict):
|
|
for sub in ("name", "full_name", "vanity", "account"):
|
|
sv = v.get(sub)
|
|
if isinstance(sv, str) and sv.strip():
|
|
return sv.strip()
|
|
return None
|
|
|
|
def _apply_sidecar(
|
|
self,
|
|
record: ImageRecord,
|
|
source: Path,
|
|
artist: Artist | None,
|
|
*,
|
|
explicit_source: Source | None = None,
|
|
) -> None:
|
|
"""Read the sidecar adjacent to `source` and wire up provenance.
|
|
|
|
If `explicit_source` is passed (FC-3c attach_in_place case), the
|
|
sidecar's post is linked to that Source instead of looking up /
|
|
creating one keyed by the sidecar's post_url. Without it, the
|
|
filesystem importer's per-post-Source creation is preserved.
|
|
"""
|
|
sc = find_sidecar(source)
|
|
if sc is None:
|
|
return
|
|
try:
|
|
data = json.loads(sc.read_text("utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError("sidecar JSON is not an object")
|
|
except Exception as exc:
|
|
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
|
return
|
|
|
|
sd = parse_sidecar(data)
|
|
|
|
if artist is None:
|
|
name = self._sidecar_artist_name(data)
|
|
if name:
|
|
artist = self._upsert_artist(name)
|
|
if artist is None:
|
|
log.warning(
|
|
"sidecar present for %s but no artist; skipping provenance",
|
|
source,
|
|
)
|
|
return
|
|
|
|
if record.artist_id is None:
|
|
record.artist_id = artist.id
|
|
|
|
if explicit_source is not None:
|
|
src = explicit_source
|
|
else:
|
|
platform = sd.platform or "unknown"
|
|
url = sd.post_url or f"sidecar:{platform}"
|
|
src = self.session.execute(
|
|
select(Source).where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == platform,
|
|
Source.url == url,
|
|
)
|
|
).scalar_one_or_none()
|
|
if src is None:
|
|
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
self.session.add(src)
|
|
self.session.flush()
|
|
|
|
epid = sd.external_post_id or sc.stem
|
|
post = self.session.execute(
|
|
select(Post).where(
|
|
Post.source_id == src.id,
|
|
Post.external_post_id == epid,
|
|
)
|
|
).scalar_one_or_none()
|
|
if post is None:
|
|
post = Post(source_id=src.id, external_post_id=epid)
|
|
self.session.add(post)
|
|
self.session.flush()
|
|
if sd.post_url is not None:
|
|
post.post_url = sd.post_url
|
|
if sd.post_title is not None:
|
|
post.post_title = sd.post_title
|
|
if sd.post_date is not None:
|
|
post.post_date = sd.post_date
|
|
if sd.description is not None:
|
|
post.description = sd.description
|
|
if sd.attachment_count is not None:
|
|
post.attachment_count = sd.attachment_count
|
|
post.raw_metadata = sd.raw
|
|
|
|
exists = self.session.execute(
|
|
select(ImageProvenance.id).where(
|
|
ImageProvenance.image_record_id == record.id,
|
|
ImageProvenance.post_id == post.id,
|
|
)
|
|
).scalar_one_or_none()
|
|
if exists is None:
|
|
self.session.add(
|
|
ImageProvenance(
|
|
image_record_id=record.id,
|
|
post_id=post.id,
|
|
source_id=src.id,
|
|
captured_metadata=sd.raw,
|
|
)
|
|
)
|
|
if record.primary_post_id is None:
|
|
record.primary_post_id = post.id
|
|
self.session.flush()
|
|
|
|
def _copy_to_library(
|
|
self, source: Path, sha: str, attribution_path: Path
|
|
) -> Path:
|
|
"""Copy `source` to its final library path. Returns the destination.
|
|
|
|
Atomic write: copies to .partial then renames. Shared by
|
|
_import_media (filesystem scan) and _supersede (when new_path is
|
|
not passed). FC-3c's attach_in_place skips this helper entirely
|
|
— the file is already at its final home.
|
|
"""
|
|
subdir = derive_subdir(attribution_path, self.import_root)
|
|
dest_dir = self.images_root / subdir if subdir else self.images_root
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
dest_name = hash_suffixed_name(source.stem, sha, source.suffix)
|
|
dest = dest_dir / dest_name
|
|
partial = dest.with_suffix(dest.suffix + ".partial")
|
|
shutil.copy2(source, partial)
|
|
partial.rename(dest)
|
|
return dest
|
|
|
|
def _supersede(
|
|
self, existing: ImageRecord, source: Path, sha: str,
|
|
phash: str, width: int | None, height: int | None,
|
|
*, new_path: Path | None = None,
|
|
) -> None:
|
|
"""Replace `existing`'s file with the larger `source`, keeping the
|
|
row id (so tags/series/curation stay attached). ML is cleared so
|
|
the import task re-derives it on the new pixels.
|
|
|
|
After the file swap, the new file's adjacent gallery-dl sidecar
|
|
(if any) is applied via _apply_sidecar — operator-flagged
|
|
2026-05-25: scanning a GS download dir with smaller IR-migrated
|
|
images on the receiving end used to swap files but lose the GS
|
|
sidecar's post metadata entirely. _apply_sidecar is additive
|
|
(find-or-create Post / Source / ImageProvenance, NULL-only
|
|
primary_post_id update) so any pre-existing Post linkage
|
|
survives untouched.
|
|
|
|
If `new_path` is provided, `source` is assumed to ALREADY be at
|
|
that path (FC-3c attach_in_place case) — skip the copy step.
|
|
Otherwise the file is copied via _copy_to_library."""
|
|
if new_path is None:
|
|
dest = self._copy_to_library(source, sha, source)
|
|
else:
|
|
dest = new_path
|
|
|
|
old_path = existing.path
|
|
old_thumb = existing.thumbnail_path
|
|
|
|
existing.path = str(dest)
|
|
existing.sha256 = sha
|
|
existing.phash = phash
|
|
existing.size_bytes = dest.stat().st_size
|
|
existing.mime = _mime_for(source)
|
|
existing.width = width
|
|
existing.height = height
|
|
existing.thumbnail_path = None
|
|
existing.integrity_status = "unknown"
|
|
existing.tagger_predictions = None
|
|
existing.tagger_model_version = None
|
|
existing.siglip_embedding = None
|
|
existing.siglip_model_version = None
|
|
existing.centroid_scores = None
|
|
# created_at intentionally preserved; updated_at auto-bumps.
|
|
self.session.flush()
|
|
self.session.commit()
|
|
|
|
# Sidecar enrichment from the new (larger) file's location.
|
|
# _apply_sidecar resolves artist from the sidecar itself if the
|
|
# existing row has none, and is internally guarded against
|
|
# missing-or-malformed sidecars (silent return).
|
|
try:
|
|
self._apply_sidecar(existing, source, None)
|
|
except Exception as exc:
|
|
# Don't unwind the supersede DB swap if sidecar parsing
|
|
# blows up unexpectedly — the file replacement is the
|
|
# critical operation, sidecar is enrichment.
|
|
log.warning(
|
|
"sidecar enrichment failed during supersede of "
|
|
"image_record.id=%s from %s: %s",
|
|
existing.id, source, exc,
|
|
)
|
|
|
|
for stale in (old_path, old_thumb):
|
|
if not stale or stale == str(dest):
|
|
# If the supersede kept the file in place (new_path == old
|
|
# location), don't delete it.
|
|
continue
|
|
try:
|
|
p = Path(stale)
|
|
if p.exists():
|
|
p.unlink()
|
|
except Exception:
|
|
# Benign orphan; the DB swap already committed. Don't undo it.
|
|
pass
|
|
|
|
def _transparency_pct(self, source: Path) -> float:
|
|
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
|
with Image.open(source) as im:
|
|
if im.mode not in ("RGBA", "LA") and not (
|
|
im.mode == "P" and "transparency" in im.info
|
|
):
|
|
return 0.0
|
|
if im.mode != "RGBA":
|
|
im = im.convert("RGBA")
|
|
alpha = im.getchannel("A")
|
|
histogram = alpha.histogram()
|
|
transparent = histogram[0]
|
|
total = sum(histogram)
|
|
return transparent / total if total else 0.0
|