bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switched to tagging and added views to support it, in progress for other tagging functions.

Browse Source
This commit is contained in:
Bryan Van Deusen
2025-08-14 21:36:27 -04:00
parent e9793ba38c
commit 3ff34ec9c2
12 changed files with 943 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/image_importer.py
+104 -52
View File
@@ -241,37 +241,33 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
return summary
def process_archive(archive_path: str, dest_dir: str, artist: str, existing_phashes) -> int:
def process_archive(archive_path, dest_dir, artist, existing_phashes):
"""
Extract archive into a temp dir, import/tag media, and record ArchiveRecord for de-dup.
Extracts archive to temp dir, imports/tag existing images, and records ArchiveRecord.
Creates an archive:* tag ONLY if at least one image was imported or tagged.
Returns number of items imported or tagged.
"""
archive_size = os.path.getsize(archive_path)
archive_hash = calculate_hash(archive_path)
# De-dup the archive itself
# Skip if archive already processed
existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
if existing_archive:
print(f"[SKIP] Archive already imported: {archive_path}")
return 0
# Create/reuse tag for this archive
tag_name = f"archive:{artist}/{os.path.basename(archive_path)}"
tag = Tag.query.filter_by(name=tag_name).first()
if not tag:
tag = Tag(name=tag_name, kind="archive")
db.session.add(tag)
db.session.flush() # ensure tag.id
tmpdir = tempfile.mkdtemp(prefix="extract_")
# Create an extraction dir under the configured temp base
tmpdir = tempfile.mkdtemp(prefix="extract_", dir=str(get_tmp_base()))
touched_records = [] # ImageRecord objects we imported or matched
imported_or_tagged = 0
created_tag = None
try:
# Resilient extraction (unar → fallback 7z for RAR; 7z → fallback unar for others)
extract_archive_resilient(archive_path, tmpdir)
# Extract using pyunpack
Archive(archive_path).extractall(tmpdir)
# Walk extracted tree and process media files
# Import / collect all media
for root, _, files in os.walk(tmpdir):
for file in files:
if not file.lower().endswith(ALLOWED_MEDIA_EXTS):
@@ -281,24 +277,41 @@ def process_archive(archive_path: str, dest_dir: str, artist: str, existing_phas
added, _phash, rec = import_single_file(
src_path=src_path,
dest_dir=dest_dir,
artist=artist,
artist=artist, # still used for auto artist:<name> tag
existing_phashes=existing_phashes,
commit=False # batch after loop
commit=False
)
if rec is not None and tag not in rec.tags:
rec.tags.append(tag)
imported_or_tagged += 1
if rec is not None:
touched_records.append(rec)
if added:
imported_or_tagged += 1
# Record this archive as processed
# Only now: create/attach the archive tag if we actually touched images
if touched_records:
created_tag = Tag.query.filter_by(name=tag_name).first()
if not created_tag:
created_tag = Tag(name=tag_name, kind='archive')
db.session.add(created_tag)
db.session.flush()
newly_tagged = 0
for rec in touched_records:
if created_tag not in rec.tags:
rec.tags.append(created_tag)
newly_tagged += 1
imported_or_tagged += newly_tagged
# Record this archive as processed.
arch = ArchiveRecord(
filename=archive_path,
file_size=archive_size,
hash=archive_hash,
artist=artist,
tag=tag
tag=created_tag # None if we had no media
)
db.session.add(arch)
db.session.commit()
print(f"[INFO] Archive processed: {archive_path} items={imported_or_tagged}")
return imported_or_tagged
@@ -307,69 +320,85 @@ def process_archive(archive_path: str, dest_dir: str, artist: str, existing_phas
# =============================================================================
# Core: Single-file import path
# Core: Single-file import path (content-addressed storage)
# =============================================================================
def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: str) -> str:
"""
Always produce a content-addressed destination:
<target>/<name_without_ext>__<hash[:10]><ext>
If that path already exists, append a numeric suffix.
"""
base, ext = os.path.splitext(original_name)
candidate = os.path.join(target_dir, f"{base}__{content_hash[:10]}{ext}")
if not os.path.exists(candidate):
return candidate
# Rare: collision (e.g., concurrent import) → add counter
i = 2
while True:
candidate = os.path.join(target_dir, f"{base}__{content_hash[:10]}_{i}{ext}")
if not os.path.exists(candidate):
return candidate
i += 1
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, commit: bool = True):
"""
Import a single media file. If an equivalent file already exists, return that
record so callers can tag it (no duplicate import).
Returns (added: bool, phash_or_None, record_or_None).
"""
filename = os.path.basename(src_path)
original_name = os.path.basename(src_path)
print(f"[INFO] Processing: {src_path}")
file_size = os.path.getsize(src_path)
# Check by name + size
existing = ImageRecord.query.filter_by(filename=filename, file_size=file_size).first()
if existing:
print(f"[SKIP] Duplicate by name+size: {filename}")
return (False, None, existing)
# Check by content hash
# Compute content hash first (authoritative de-dup + used in filepath)
file_hash = calculate_hash(src_path)
existing = ImageRecord.query.filter_by(hash=file_hash).first()
if existing:
print(f"[SKIP] Duplicate by hash: {filename}")
print(f"[SKIP] Duplicate by hash: {original_name}")
return (False, None, existing)
# Optional info: same name+size (but different content) → we still import with hashed name
existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
if existing_ns:
print(f"[INFO] Same name+size exists but different hash; storing with hashed name: {original_name}")
# Metadata & pHash similarity
metadata = extract_metadata(src_path)
if not metadata["width"] or not metadata["height"]:
print(f"[SKIP] Missing dimension data for {filename}")
print(f"[SKIP] Missing dimension data for {original_name}")
return (False, None, None)
phash = calculate_perceptual_hash(src_path)
if phash and is_similar_image(phash, metadata["width"], metadata["height"], existing_phashes):
print(f"[SKIP] {filename} is visually similar to an existing larger image.")
print(f"[SKIP] {original_name} is visually similar to an existing larger image.")
return (False, phash, None)
# Copy into /images/<artist>/<filename>
# Copy into /images/<artist>/<base>__<hash[:10]><ext>
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = os.path.join(target_dir, filename)
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
shutil.copy2(src_path, dest_path)
# Thumbnail
# Thumbnails (mirrored for both images and videos)
try:
if dest_path.lower().endswith((".mp4", ".mov")):
thumb_filename = f"{uuid.uuid4().hex}.jpg"
thumb_path = os.path.join(dest_dir, "thumbs", thumb_filename)
thumb_path = generate_video_thumbnail(dest_path, thumb_path)
thumb_path = generate_video_thumbnail_mirrored(dest_path)
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
print(f"[WARN] Failed to generate thumbnail for {filename}: {e}")
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
thumb_path = None
# Create DB record
# Create DB record (keep display name as original; filepath is content-addressed)
record = ImageRecord(
filename=filename,
filename=original_name,
filepath=dest_path,
thumb_path=thumb_path,
hash=file_hash,
perceptual_hash=str(phash) if phash else None,
artist=artist,
file_size=metadata["file_size"],
width=metadata["width"],
height=metadata["height"],
@@ -378,6 +407,19 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
taken_at=metadata["taken_at"],
imported_at=datetime.utcnow()
)
# Auto-tag with artist:<name> (treat artist as a tag only)
if artist:
artist_tag_name = f"artist:{artist}"
artist_tag = Tag.query.filter_by(name=artist_tag_name).first()
if not artist_tag:
artist_tag = Tag(name=artist_tag_name, kind="artist")
db.session.add(artist_tag)
db.session.flush()
if artist_tag not in record.tags:
record.tags.append(artist_tag)
db.session.add(record)
if commit:
db.session.commit()
@@ -390,13 +432,15 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
def generate_thumbnail(image_path: str, size: tuple[int, int] = (400, 400), overwrite: bool = False) -> str:
"""
Save thumbnails mirrored under /images/thumbs/<artist>/<filename>.
Save thumbnails mirrored under /images/thumbs/<artist>/<unique_filename>.
(unique_filename includes the hash suffix, so no collisions)
"""
images_root = Path("/images").resolve()
image_path = Path(image_path).resolve()
rel_path = image_path.relative_to(images_root) # raises if not under /images
thumb_path = images_root / "thumbs" / rel_path
thumb_path = thumb_path.with_suffix(".jpg") if thumb_path.suffix.lower() not in (".jpg", ".jpeg") else thumb_path
thumb_path.parent.mkdir(parents=True, exist_ok=True)
if not overwrite and thumb_path.exists():
@@ -409,15 +453,23 @@ def generate_thumbnail(image_path: str, size: tuple[int, int] = (400, 400), over
return str(thumb_path)
def generate_video_thumbnail(video_path: str, output_path: str, time_position: str = "00:00:01") -> str | None:
os.makedirs(os.path.dirname(output_path), exist_ok=True)
command = [
"ffmpeg", "-i", video_path, "-ss", time_position,
"-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", output_path
def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:00:01") -> str | None:
"""
Mirror the stored (hashed) video path under /images/thumbs/... and save as .jpg.
"""
images_root = Path("/images").resolve()
video_path = Path(video_path).resolve()
rel_path = video_path.relative_to(images_root) # raises if not under /images
thumb_path = (images_root / "thumbs" / rel_path).with_suffix(".jpg")
thumb_path.parent.mkdir(parents=True, exist_ok=True)
cmd = [
"ffmpeg", "-i", str(video_path), "-ss", time_position,
"-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", str(thumb_path)
]
try:
subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return output_path
subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return str(thumb_path)
except subprocess.CalledProcessError as e:
print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
return None
@@ -544,4 +596,4 @@ def is_first_volume(path: str) -> bool:
if name.endswith((".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
return True
return True
return True