Add image search with local cache (Phase 23)
Images found via SearXNG are fetched server-side, stored on disk, and served from /api/images/<id> — the user's browser never contacts the original image host. Original URLs are preserved for citation. New files: - alembic/versions/0016_add_image_cache.py — image_cache table - src/fabledassistant/models/image_cache.py — SQLAlchemy model - src/fabledassistant/services/images.py — fetch/store/serve logic - src/fabledassistant/routes/images.py — GET /api/images/<id> Modified: - config.py: IMAGE_CACHE_DIR (/data/images), IMAGE_MAX_BYTES (5 MB) - research.py: _search_searxng_images() — SearXNG categories=images - tools.py: _IMAGE_TOOLS def + search_images branch in execute_tool - intent.py: search_images routing rule (explicit visual language only) - app.py: register images_bp - docker-compose.yml: image_cache named volume mounted at /data/images - ToolCallCard.vue: "image_search" label Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
"""Image cache service: fetch from web, store on disk, serve locally.
|
||||
|
||||
Images are stored in IMAGE_CACHE_DIR (default /data/images) and tracked in
|
||||
the image_cache DB table. The original URL is preserved for citation.
|
||||
Deduplication is by SHA-256 of the original URL — the same image requested
|
||||
twice shares one file and one DB row.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.image_cache import ImageCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ALLOWED_TYPES = {
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/webp",
|
||||
"image/avif",
|
||||
}
|
||||
|
||||
_EXT_MAP = {
|
||||
"image/jpeg": "jpg",
|
||||
"image/png": "png",
|
||||
"image/gif": "gif",
|
||||
"image/webp": "webp",
|
||||
"image/avif": "avif",
|
||||
}
|
||||
|
||||
|
||||
def _cache_dir() -> Path:
|
||||
d = Path(Config.IMAGE_CACHE_DIR)
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def _url_hash(url: str) -> str:
|
||||
return hashlib.sha256(url.encode()).hexdigest()
|
||||
|
||||
|
||||
async def fetch_and_store_image(
|
||||
url: str,
|
||||
title: str | None = None,
|
||||
source_domain: str | None = None,
|
||||
) -> ImageCache | None:
|
||||
"""Fetch an image URL, write it to disk, persist metadata to DB.
|
||||
|
||||
Returns the ImageCache record (new or existing).
|
||||
Returns None if the URL is unreachable, not a valid image type, or
|
||||
exceeds IMAGE_MAX_BYTES.
|
||||
"""
|
||||
url_hash = _url_hash(url)
|
||||
|
||||
# Return existing record if already cached
|
||||
async with async_session() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(ImageCache).where(ImageCache.url_hash == url_hash)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
logger.debug("Image cache hit id=%d for %s", existing.id, url[:80])
|
||||
return existing
|
||||
|
||||
# Infer source domain from URL if not supplied
|
||||
if not source_domain:
|
||||
try:
|
||||
source_domain = urlparse(url).netloc or None
|
||||
except Exception:
|
||||
source_domain = None
|
||||
|
||||
# Fetch from origin
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0)",
|
||||
}
|
||||
# Provide a same-origin Referer to bypass basic hotlink protection
|
||||
if source_domain:
|
||||
headers["Referer"] = f"https://{source_domain}/"
|
||||
resp = await client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
|
||||
ct = resp.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if ct not in _ALLOWED_TYPES:
|
||||
logger.warning("Skipping non-image content-type '%s' for %s", ct, url[:80])
|
||||
return None
|
||||
|
||||
# Reject oversized images before buffering all bytes
|
||||
cl = resp.headers.get("content-length")
|
||||
if cl and int(cl) > Config.IMAGE_MAX_BYTES:
|
||||
logger.warning("Image too large (%s bytes, limit %d) at %s", cl, Config.IMAGE_MAX_BYTES, url[:80])
|
||||
return None
|
||||
|
||||
data = resp.content
|
||||
if len(data) > Config.IMAGE_MAX_BYTES:
|
||||
logger.warning("Image too large (%d bytes) at %s", len(data), url[:80])
|
||||
return None
|
||||
if not data:
|
||||
return None
|
||||
|
||||
except Exception:
|
||||
logger.warning("Failed to fetch image from %s", url[:80], exc_info=True)
|
||||
return None
|
||||
|
||||
# Write to disk
|
||||
file_ext = _EXT_MAP.get(ct, "jpg")
|
||||
file_path = _cache_dir() / f"{url_hash}.{file_ext}"
|
||||
try:
|
||||
file_path.write_bytes(data)
|
||||
except Exception:
|
||||
logger.warning("Failed to write image to disk: %s", file_path, exc_info=True)
|
||||
return None
|
||||
|
||||
# Persist metadata — handle the rare race-condition duplicate gracefully
|
||||
record = ImageCache(
|
||||
url_hash=url_hash,
|
||||
original_url=url,
|
||||
source_domain=source_domain,
|
||||
title=title,
|
||||
content_type=ct,
|
||||
file_size=len(data),
|
||||
file_ext=file_ext,
|
||||
)
|
||||
async with async_session() as session:
|
||||
try:
|
||||
session.add(record)
|
||||
await session.commit()
|
||||
await session.refresh(record)
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(ImageCache).where(ImageCache.url_hash == url_hash)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
return existing
|
||||
logger.warning("Failed to persist image cache record for %s", url[:80], exc_info=True)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"Cached image id=%d (%s, %d bytes) from %s",
|
||||
record.id, ct, len(data), url[:80],
|
||||
)
|
||||
return record
|
||||
|
||||
|
||||
async def get_image_record(image_id: int) -> ImageCache | None:
|
||||
"""Return the ImageCache row for a given ID, or None."""
|
||||
async with async_session() as session:
|
||||
return (
|
||||
await session.execute(
|
||||
select(ImageCache).where(ImageCache.id == image_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def get_image_path(record: ImageCache) -> Path:
|
||||
"""Filesystem path for a cached image record."""
|
||||
return _cache_dir() / f"{record.url_hash}.{record.file_ext}"
|
||||
@@ -106,6 +106,10 @@ Rules:
|
||||
the internet. Do NOT use for creative questions, brainstorming, game design, writing help, or
|
||||
when the user is building on content they already have. Do NOT use when the user references
|
||||
their own notes or prior research — use search_notes instead.
|
||||
- search_images: ONLY when the user explicitly asks to SEE, SHOW, DISPLAY, or VIEW an image or
|
||||
photo. Trigger phrases: "show me a picture/photo/image of", "what does X look like",
|
||||
"find a photo of", "display an image of". Do NOT use for general questions about appearance,
|
||||
descriptions, or when the user just wants information without visual content.
|
||||
- research_topic: user wants a comprehensive, multi-section research note created from web sources.
|
||||
Use whenever the user wants to deeply understand, learn about, or get a full written reference
|
||||
on any subject — regardless of how they phrase it. The topic can be anything: technical subjects,
|
||||
|
||||
@@ -184,6 +184,52 @@ async def _search_searxng(query: str) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
async def _search_searxng_images(query: str) -> list[dict]:
|
||||
"""Search SearXNG image category and return [{img_src, page_url, title, source_domain}]."""
|
||||
url = Config.SEARXNG_URL.rstrip("/") + "/search"
|
||||
params = {"q": query, "format": "json", "categories": "images"}
|
||||
for attempt in range(3):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
if resp.status_code == 429:
|
||||
retry_after = int(resp.headers.get("Retry-After", "5"))
|
||||
wait = min(retry_after, 10) * (attempt + 1)
|
||||
logger.warning(
|
||||
"SearXNG image 429 for '%s' (attempt %d/3), waiting %ds",
|
||||
query, attempt + 1, wait,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
out = []
|
||||
for r in data.get("results", []):
|
||||
img_src = r.get("img_src") or r.get("thumbnail_src", "")
|
||||
if not img_src:
|
||||
continue
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
source_domain = urlparse(r.get("url", "")).netloc or ""
|
||||
except Exception:
|
||||
source_domain = ""
|
||||
out.append({
|
||||
"img_src": img_src,
|
||||
"page_url": r.get("url", ""),
|
||||
"title": r.get("title", ""),
|
||||
"source_domain": source_domain,
|
||||
})
|
||||
return out
|
||||
except httpx.HTTPStatusError:
|
||||
logger.warning("SearXNG image search failed for '%s'", query, exc_info=True)
|
||||
return []
|
||||
except Exception:
|
||||
logger.warning("SearXNG image search failed for '%s'", query, exc_info=True)
|
||||
return []
|
||||
logger.warning("SearXNG image search gave up after 3 attempts for '%s'", query)
|
||||
return []
|
||||
|
||||
|
||||
async def _synthesize_note(
|
||||
topic: str,
|
||||
sources: list[dict],
|
||||
|
||||
@@ -710,6 +710,33 @@ _RESEARCH_TOOLS = [
|
||||
}
|
||||
]
|
||||
|
||||
_IMAGE_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_images",
|
||||
"description": (
|
||||
"Search the web for images and display them inline in the response. "
|
||||
"ONLY use this when the user explicitly asks to SEE, SHOW, VIEW, or DISPLAY "
|
||||
"an image or photo — e.g. 'show me a picture of X', 'what does X look like', "
|
||||
"'find a photo of X', 'display an image of X'. "
|
||||
"Do NOT use for general descriptions, factual questions, or when the user just "
|
||||
"wants information without visual content. Returns at most 2 images."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The image search query",
|
||||
}
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||
"""Build the tool list for a user based on their configured integrations."""
|
||||
@@ -719,6 +746,7 @@ async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||
if Config.searxng_enabled():
|
||||
tools.extend(_SEARCH_TOOLS)
|
||||
tools.extend(_RESEARCH_TOOLS)
|
||||
tools.extend(_IMAGE_TOOLS)
|
||||
logger.debug("User %d: %d tools available", user_id, len(tools))
|
||||
return tools
|
||||
|
||||
@@ -1215,6 +1243,41 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
|
||||
"data": {"topic": topic},
|
||||
}
|
||||
|
||||
elif tool_name == "search_images":
|
||||
from fabledassistant.services.images import fetch_and_store_image
|
||||
from fabledassistant.services.research import _search_searxng_images
|
||||
query = arguments.get("query", "")
|
||||
if not query:
|
||||
return {"success": False, "error": "query is required"}
|
||||
raw_results = await _search_searxng_images(query)
|
||||
if not raw_results:
|
||||
return {"success": False, "error": f"No images found for '{query}'"}
|
||||
# Fetch and cache up to 2 images
|
||||
images = []
|
||||
for r in raw_results[:2]:
|
||||
img_url = r.get("img_src", "")
|
||||
if not img_url:
|
||||
continue
|
||||
record = await fetch_and_store_image(
|
||||
url=img_url,
|
||||
title=r.get("title"),
|
||||
source_domain=r.get("source_domain"),
|
||||
)
|
||||
if record:
|
||||
images.append({
|
||||
"local_url": f"/api/images/{record.id}",
|
||||
"page_url": r.get("page_url", ""),
|
||||
"source_domain": r.get("source_domain", ""),
|
||||
"title": record.title or query,
|
||||
})
|
||||
if not images:
|
||||
return {"success": False, "error": f"Could not retrieve images for '{query}'"}
|
||||
return {
|
||||
"success": True,
|
||||
"type": "image_search",
|
||||
"data": {"query": query, "images": images},
|
||||
}
|
||||
|
||||
else:
|
||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user