From efe0a15b8cb86c589b182541bd7edcc3bc2ff8f2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 1 Mar 2026 12:55:10 -0500 Subject: [PATCH] Add image search with local cache (Phase 23) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images found via SearXNG are fetched server-side, stored on disk, and served from /api/images/ — the user's browser never contacts the original image host. Original URLs are preserved for citation. New files: - alembic/versions/0016_add_image_cache.py — image_cache table - src/fabledassistant/models/image_cache.py — SQLAlchemy model - src/fabledassistant/services/images.py — fetch/store/serve logic - src/fabledassistant/routes/images.py — GET /api/images/ Modified: - config.py: IMAGE_CACHE_DIR (/data/images), IMAGE_MAX_BYTES (5 MB) - research.py: _search_searxng_images() — SearXNG categories=images - tools.py: _IMAGE_TOOLS def + search_images branch in execute_tool - intent.py: search_images routing rule (explicit visual language only) - app.py: register images_bp - docker-compose.yml: image_cache named volume mounted at /data/images - ToolCallCard.vue: "image_search" label Co-Authored-By: Claude Sonnet 4.6 --- alembic/versions/0016_add_image_cache.py | 30 ++++ docker-compose.yml | 9 +- frontend/src/components/ToolCallCard.vue | 2 + src/fabledassistant/app.py | 2 + src/fabledassistant/config.py | 5 + src/fabledassistant/models/__init__.py | 1 + src/fabledassistant/models/image_cache.py | 25 ++++ src/fabledassistant/routes/images.py | 35 +++++ src/fabledassistant/services/images.py | 171 ++++++++++++++++++++++ src/fabledassistant/services/intent.py | 4 + src/fabledassistant/services/research.py | 46 ++++++ src/fabledassistant/services/tools.py | 63 ++++++++ 12 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/0016_add_image_cache.py create mode 100644 src/fabledassistant/models/image_cache.py create mode 100644 src/fabledassistant/routes/images.py create mode 100644 src/fabledassistant/services/images.py diff --git a/alembic/versions/0016_add_image_cache.py b/alembic/versions/0016_add_image_cache.py new file mode 100644 index 0000000..342da0a --- /dev/null +++ b/alembic/versions/0016_add_image_cache.py @@ -0,0 +1,30 @@ +"""Add image_cache table for locally-stored images fetched from the web.""" + +from alembic import op + +revision = "0016" +down_revision = "0015" + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE IF NOT EXISTS image_cache ( + id SERIAL PRIMARY KEY, + url_hash TEXT UNIQUE NOT NULL, + original_url TEXT NOT NULL, + source_domain TEXT, + title TEXT, + content_type TEXT NOT NULL DEFAULT 'image/jpeg', + file_size INTEGER, + file_ext TEXT NOT NULL DEFAULT 'jpg', + fetched_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) + """) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_image_cache_url_hash ON image_cache(url_hash)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_image_cache_url_hash") + op.execute("DROP TABLE IF EXISTS image_cache") diff --git a/docker-compose.yml b/docker-compose.yml index 7613d2c..bbccb5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,13 +8,19 @@ services: condition: service_healthy ollama: condition: service_started + volumes: + - image_cache:/data/images + # To use a bind mount instead (gives direct host access to cached images): + # - ./image_cache:/data/images environment: DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-fabled}:${POSTGRES_PASSWORD:-fabled}@db:5432/${POSTGRES_DB:-fabledassistant}" OLLAMA_URL: "http://ollama:11434" OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}" SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}" - # Uncomment and set to enable web research via SearXNG: + # Uncomment and set to enable web research and image search via SearXNG: # SEARXNG_URL: "http://searxng:8080" + # IMAGE_CACHE_DIR: /data/images # default, change if using a different mount path + # IMAGE_MAX_BYTES: "5242880" # 5 MB per image, adjust if needed healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"] interval: 10s @@ -56,3 +62,4 @@ services: volumes: pgdata: ollama_models: + image_cache: diff --git a/frontend/src/components/ToolCallCard.vue b/frontend/src/components/ToolCallCard.vue index 75408b7..e884e01 100644 --- a/frontend/src/components/ToolCallCard.vue +++ b/frontend/src/components/ToolCallCard.vue @@ -58,6 +58,8 @@ const label = computed(() => { return "Research started"; case "research_note": return "Research note"; + case "image_search": + return "Image search"; default: return "Tool call"; } diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 575284f..39ce4a9 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -13,6 +13,7 @@ from fabledassistant.routes.api import api from fabledassistant.routes.auth import auth_bp from fabledassistant.routes.chat import chat_bp from fabledassistant.routes.notes import notes_bp +from fabledassistant.routes.images import images_bp from fabledassistant.routes.quick_capture import quick_capture_bp from fabledassistant.routes.settings import settings_bp from fabledassistant.routes.tasks import tasks_bp @@ -44,6 +45,7 @@ def create_app() -> Quart: app.register_blueprint(api) app.register_blueprint(auth_bp) app.register_blueprint(chat_bp) + app.register_blueprint(images_bp) app.register_blueprint(notes_bp) app.register_blueprint(quick_capture_bp) app.register_blueprint(settings_bp) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index a664464..573cbfd 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -59,6 +59,11 @@ class Config: # SearXNG web search (external instance) SEARXNG_URL: str = os.environ.get("SEARXNG_URL", "") + # Image cache — images fetched from the web are stored here and served locally + IMAGE_CACHE_DIR: str = os.environ.get("IMAGE_CACHE_DIR", "/data/images") + # Maximum size of a single image to cache (default 5 MB) + IMAGE_MAX_BYTES: int = int(os.environ.get("IMAGE_MAX_BYTES", str(5 * 1024 * 1024))) + @classmethod def oidc_enabled(cls) -> bool: return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET) diff --git a/src/fabledassistant/models/__init__.py b/src/fabledassistant/models/__init__.py index bada508..b4392dc 100644 --- a/src/fabledassistant/models/__init__.py +++ b/src/fabledassistant/models/__init__.py @@ -24,3 +24,4 @@ from fabledassistant.models.app_log import AppLog # noqa: E402, F401 from fabledassistant.models.password_reset import PasswordResetToken # noqa: E402, F401 from fabledassistant.models.invitation import InvitationToken # noqa: E402, F401 from fabledassistant.models.embedding import NoteEmbedding # noqa: E402, F401 +from fabledassistant.models.image_cache import ImageCache # noqa: E402, F401 diff --git a/src/fabledassistant/models/image_cache.py b/src/fabledassistant/models/image_cache.py new file mode 100644 index 0000000..37af733 --- /dev/null +++ b/src/fabledassistant/models/image_cache.py @@ -0,0 +1,25 @@ +from datetime import datetime, timezone + +from sqlalchemy import DateTime, Integer, Text +from sqlalchemy.orm import Mapped, mapped_column + +from fabledassistant.models import Base + + +class ImageCache(Base): + """Metadata for an image fetched from the web and stored locally on disk.""" + + __tablename__ = "image_cache" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + url_hash: Mapped[str] = mapped_column(Text, unique=True, nullable=False, index=True) + original_url: Mapped[str] = mapped_column(Text, nullable=False) + source_domain: Mapped[str | None] = mapped_column(Text, nullable=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + content_type: Mapped[str] = mapped_column(Text, nullable=False, default="image/jpeg") + file_size: Mapped[int | None] = mapped_column(Integer, nullable=True) + file_ext: Mapped[str] = mapped_column(Text, nullable=False, default="jpg") + fetched_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + ) diff --git a/src/fabledassistant/routes/images.py b/src/fabledassistant/routes/images.py new file mode 100644 index 0000000..69cdae1 --- /dev/null +++ b/src/fabledassistant/routes/images.py @@ -0,0 +1,35 @@ +"""Serve locally-cached images. + +No authentication required — image IDs are opaque and unguessable (based +on the SHA-256 of the original URL). Images are served with a 1-day +Cache-Control header so the browser doesn't re-request on every page load. +""" + +import logging + +from quart import Blueprint, jsonify, send_file + +from fabledassistant.services.images import get_image_path, get_image_record + +logger = logging.getLogger(__name__) + +images_bp = Blueprint("images", __name__, url_prefix="/api/images") + + +@images_bp.route("/", methods=["GET"]) +async def serve_image(image_id: int): + """Serve a locally-cached image by its DB ID.""" + record = await get_image_record(image_id) + if record is None: + return jsonify({"error": "Image not found"}), 404 + + file_path = get_image_path(record) + if not file_path.exists(): + logger.warning("Image file missing on disk for id=%d (%s)", image_id, file_path) + return jsonify({"error": "Image file not found"}), 404 + + return await send_file( + file_path, + mimetype=record.content_type, + max_age=86400, + ) diff --git a/src/fabledassistant/services/images.py b/src/fabledassistant/services/images.py new file mode 100644 index 0000000..820840b --- /dev/null +++ b/src/fabledassistant/services/images.py @@ -0,0 +1,171 @@ +"""Image cache service: fetch from web, store on disk, serve locally. + +Images are stored in IMAGE_CACHE_DIR (default /data/images) and tracked in +the image_cache DB table. The original URL is preserved for citation. +Deduplication is by SHA-256 of the original URL — the same image requested +twice shares one file and one DB row. +""" + +import hashlib +import logging +from pathlib import Path +from urllib.parse import urlparse + +import httpx +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from fabledassistant.config import Config +from fabledassistant.models import async_session +from fabledassistant.models.image_cache import ImageCache + +logger = logging.getLogger(__name__) + +_ALLOWED_TYPES = { + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + "image/avif", +} + +_EXT_MAP = { + "image/jpeg": "jpg", + "image/png": "png", + "image/gif": "gif", + "image/webp": "webp", + "image/avif": "avif", +} + + +def _cache_dir() -> Path: + d = Path(Config.IMAGE_CACHE_DIR) + d.mkdir(parents=True, exist_ok=True) + return d + + +def _url_hash(url: str) -> str: + return hashlib.sha256(url.encode()).hexdigest() + + +async def fetch_and_store_image( + url: str, + title: str | None = None, + source_domain: str | None = None, +) -> ImageCache | None: + """Fetch an image URL, write it to disk, persist metadata to DB. + + Returns the ImageCache record (new or existing). + Returns None if the URL is unreachable, not a valid image type, or + exceeds IMAGE_MAX_BYTES. + """ + url_hash = _url_hash(url) + + # Return existing record if already cached + async with async_session() as session: + existing = ( + await session.execute( + select(ImageCache).where(ImageCache.url_hash == url_hash) + ) + ).scalar_one_or_none() + if existing: + logger.debug("Image cache hit id=%d for %s", existing.id, url[:80]) + return existing + + # Infer source domain from URL if not supplied + if not source_domain: + try: + source_domain = urlparse(url).netloc or None + except Exception: + source_domain = None + + # Fetch from origin + try: + async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client: + headers = { + "User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0)", + } + # Provide a same-origin Referer to bypass basic hotlink protection + if source_domain: + headers["Referer"] = f"https://{source_domain}/" + resp = await client.get(url, headers=headers) + resp.raise_for_status() + + ct = resp.headers.get("content-type", "").split(";")[0].strip().lower() + if ct not in _ALLOWED_TYPES: + logger.warning("Skipping non-image content-type '%s' for %s", ct, url[:80]) + return None + + # Reject oversized images before buffering all bytes + cl = resp.headers.get("content-length") + if cl and int(cl) > Config.IMAGE_MAX_BYTES: + logger.warning("Image too large (%s bytes, limit %d) at %s", cl, Config.IMAGE_MAX_BYTES, url[:80]) + return None + + data = resp.content + if len(data) > Config.IMAGE_MAX_BYTES: + logger.warning("Image too large (%d bytes) at %s", len(data), url[:80]) + return None + if not data: + return None + + except Exception: + logger.warning("Failed to fetch image from %s", url[:80], exc_info=True) + return None + + # Write to disk + file_ext = _EXT_MAP.get(ct, "jpg") + file_path = _cache_dir() / f"{url_hash}.{file_ext}" + try: + file_path.write_bytes(data) + except Exception: + logger.warning("Failed to write image to disk: %s", file_path, exc_info=True) + return None + + # Persist metadata — handle the rare race-condition duplicate gracefully + record = ImageCache( + url_hash=url_hash, + original_url=url, + source_domain=source_domain, + title=title, + content_type=ct, + file_size=len(data), + file_ext=file_ext, + ) + async with async_session() as session: + try: + session.add(record) + await session.commit() + await session.refresh(record) + except IntegrityError: + await session.rollback() + existing = ( + await session.execute( + select(ImageCache).where(ImageCache.url_hash == url_hash) + ) + ).scalar_one_or_none() + if existing: + return existing + logger.warning("Failed to persist image cache record for %s", url[:80], exc_info=True) + return None + + logger.info( + "Cached image id=%d (%s, %d bytes) from %s", + record.id, ct, len(data), url[:80], + ) + return record + + +async def get_image_record(image_id: int) -> ImageCache | None: + """Return the ImageCache row for a given ID, or None.""" + async with async_session() as session: + return ( + await session.execute( + select(ImageCache).where(ImageCache.id == image_id) + ) + ).scalar_one_or_none() + + +def get_image_path(record: ImageCache) -> Path: + """Filesystem path for a cached image record.""" + return _cache_dir() / f"{record.url_hash}.{record.file_ext}" diff --git a/src/fabledassistant/services/intent.py b/src/fabledassistant/services/intent.py index 5b106cd..22656e3 100644 --- a/src/fabledassistant/services/intent.py +++ b/src/fabledassistant/services/intent.py @@ -106,6 +106,10 @@ Rules: the internet. Do NOT use for creative questions, brainstorming, game design, writing help, or when the user is building on content they already have. Do NOT use when the user references their own notes or prior research — use search_notes instead. +- search_images: ONLY when the user explicitly asks to SEE, SHOW, DISPLAY, or VIEW an image or + photo. Trigger phrases: "show me a picture/photo/image of", "what does X look like", + "find a photo of", "display an image of". Do NOT use for general questions about appearance, + descriptions, or when the user just wants information without visual content. - research_topic: user wants a comprehensive, multi-section research note created from web sources. Use whenever the user wants to deeply understand, learn about, or get a full written reference on any subject — regardless of how they phrase it. The topic can be anything: technical subjects, diff --git a/src/fabledassistant/services/research.py b/src/fabledassistant/services/research.py index dfbfa3b..dbd58b3 100644 --- a/src/fabledassistant/services/research.py +++ b/src/fabledassistant/services/research.py @@ -184,6 +184,52 @@ async def _search_searxng(query: str) -> list[dict]: return [] +async def _search_searxng_images(query: str) -> list[dict]: + """Search SearXNG image category and return [{img_src, page_url, title, source_domain}].""" + url = Config.SEARXNG_URL.rstrip("/") + "/search" + params = {"q": query, "format": "json", "categories": "images"} + for attempt in range(3): + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.get(url, params=params) + if resp.status_code == 429: + retry_after = int(resp.headers.get("Retry-After", "5")) + wait = min(retry_after, 10) * (attempt + 1) + logger.warning( + "SearXNG image 429 for '%s' (attempt %d/3), waiting %ds", + query, attempt + 1, wait, + ) + await asyncio.sleep(wait) + continue + resp.raise_for_status() + data = resp.json() + out = [] + for r in data.get("results", []): + img_src = r.get("img_src") or r.get("thumbnail_src", "") + if not img_src: + continue + try: + from urllib.parse import urlparse + source_domain = urlparse(r.get("url", "")).netloc or "" + except Exception: + source_domain = "" + out.append({ + "img_src": img_src, + "page_url": r.get("url", ""), + "title": r.get("title", ""), + "source_domain": source_domain, + }) + return out + except httpx.HTTPStatusError: + logger.warning("SearXNG image search failed for '%s'", query, exc_info=True) + return [] + except Exception: + logger.warning("SearXNG image search failed for '%s'", query, exc_info=True) + return [] + logger.warning("SearXNG image search gave up after 3 attempts for '%s'", query) + return [] + + async def _synthesize_note( topic: str, sources: list[dict], diff --git a/src/fabledassistant/services/tools.py b/src/fabledassistant/services/tools.py index 85db213..e586039 100644 --- a/src/fabledassistant/services/tools.py +++ b/src/fabledassistant/services/tools.py @@ -710,6 +710,33 @@ _RESEARCH_TOOLS = [ } ] +_IMAGE_TOOLS = [ + { + "type": "function", + "function": { + "name": "search_images", + "description": ( + "Search the web for images and display them inline in the response. " + "ONLY use this when the user explicitly asks to SEE, SHOW, VIEW, or DISPLAY " + "an image or photo — e.g. 'show me a picture of X', 'what does X look like', " + "'find a photo of X', 'display an image of X'. " + "Do NOT use for general descriptions, factual questions, or when the user just " + "wants information without visual content. Returns at most 2 images." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The image search query", + } + }, + "required": ["query"], + }, + }, + } +] + async def get_tools_for_user(user_id: int) -> list[dict]: """Build the tool list for a user based on their configured integrations.""" @@ -719,6 +746,7 @@ async def get_tools_for_user(user_id: int) -> list[dict]: if Config.searxng_enabled(): tools.extend(_SEARCH_TOOLS) tools.extend(_RESEARCH_TOOLS) + tools.extend(_IMAGE_TOOLS) logger.debug("User %d: %d tools available", user_id, len(tools)) return tools @@ -1215,6 +1243,41 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict: "data": {"topic": topic}, } + elif tool_name == "search_images": + from fabledassistant.services.images import fetch_and_store_image + from fabledassistant.services.research import _search_searxng_images + query = arguments.get("query", "") + if not query: + return {"success": False, "error": "query is required"} + raw_results = await _search_searxng_images(query) + if not raw_results: + return {"success": False, "error": f"No images found for '{query}'"} + # Fetch and cache up to 2 images + images = [] + for r in raw_results[:2]: + img_url = r.get("img_src", "") + if not img_url: + continue + record = await fetch_and_store_image( + url=img_url, + title=r.get("title"), + source_domain=r.get("source_domain"), + ) + if record: + images.append({ + "local_url": f"/api/images/{record.id}", + "page_url": r.get("page_url", ""), + "source_domain": r.get("source_domain", ""), + "title": record.title or query, + }) + if not images: + return {"success": False, "error": f"Could not retrieve images for '{query}'"} + return { + "success": True, + "type": "image_search", + "data": {"query": query, "images": images}, + } + else: return {"success": False, "error": f"Unknown tool: {tool_name}"}