diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index cb247cf..9a31d7c 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -92,28 +92,25 @@ jobs: - run: npm run test:unit - run: npm run build - # Integration suite split into THREE parallel shards (2026-05-25, runner - # capacity bumped 2→6). Each shard gets its own Postgres + Redis service - # set and runs alembic + a disjoint subset of integration tests. Shards - # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py - # stays single-threaded per shard but multiple shards run in parallel - # wall-clock. Approximate split — rebalance once --durations=15 output - # reveals which shard is the long pole. + # Single integration job — collapsed from a 3-way shard split on 2026-06-04. + # The shards existed to parallelize ~8.5min of integration tests; once the + # throwaway Postgres runs with fsync OFF (the durability step below) the whole + # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead + # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6 + # runner slots for no wall-clock gain. One job now: spin up once, install + # once, migrate once, run every integration test. # - # Each shard's docker-ps filter uses its own unique job name to scope - # service-container resolution. act_runner appears to strip underscores - # from job names when building container labels — `int_api` yielded - # zero matches on 2026-05-25 — so shards use no-separator names - # (`intapi`, `intimp`, `intcore`) instead. Each step prints - # `docker ps -a` first so a future naming-convention shift surfaces in - # the log without another guess-and-push cycle. + # The docker-ps filter scopes to THIS job's own Postgres/Redis service + # containers by job name. act_runner strips underscores from job names when + # labelling containers (`int_api` matched nothing on 2026-05-25), so the name + # stays separator-free (`integration`). The step prints `docker ps -a` first + # so a future naming-convention shift surfaces in the log without a + # guess-and-push cycle. # - # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT - # done — per ci-requirements.md, FC is the only Python consumer of that - # image and the CI-Runner project's "add deps to image when used by >1 - # project" rule keeps the install per-job. - - intapi: + # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done — + # per ci-requirements.md, FC is the only Python consumer of that image and the + # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job. + integration: runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -144,14 +141,14 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: API integration shard (resolve service IPs, migrate, test) + - name: Integration suite (resolve service IPs, migrate, test) run: | set -eux echo "=== container landscape (diagnostic for filter scoping) ===" docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' echo "=== end landscape ===" - PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1) + PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") @@ -168,130 +165,14 @@ jobs: else pip install -r requirements.txt pytest pytest-asyncio fi + # Relax durability on the throwaway CI Postgres so the per-test + # TRUNCATE's commit-fsync — the integration teardown's dominant cost + # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is + # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit + # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with + # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms + # surprise can't red the job; fabledcurator is the postgres image's + # bootstrap superuser. + python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' alembic upgrade head - pytest tests/test_api_*.py -v -m integration --durations=15 - - intimp: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - DB_USER: fabledcurator - DB_PASSWORD: ci_integration - DB_PORT: "5432" - DB_NAME: fabledcurator_test - SECRET_KEY: ci_integration_placeholder - services: - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: fabledcurator - POSTGRES_PASSWORD: ci_integration - POSTGRES_DB: fabledcurator_test - options: >- - --health-cmd "pg_isready -U fabledcurator" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@v4 - - name: Importer integration shard (resolve service IPs, migrate, test) - run: | - set -eux - echo "=== container landscape (diagnostic for filter scoping) ===" - docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' - echo "=== end landscape ===" - PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" - PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") - RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") - test -n "$PG_IP" && test -n "$RD_IP" - export DB_HOST="$PG_IP" - export CELERY_BROKER_URL="redis://$RD_IP:6379/0" - export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - for i in $(seq 1 60); do - (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break - sleep 2 - done - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - alembic upgrade head - pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15 - - intcore: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - DB_USER: fabledcurator - DB_PASSWORD: ci_integration - DB_PORT: "5432" - DB_NAME: fabledcurator_test - SECRET_KEY: ci_integration_placeholder - services: - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: fabledcurator - POSTGRES_PASSWORD: ci_integration - POSTGRES_DB: fabledcurator_test - options: >- - --health-cmd "pg_isready -U fabledcurator" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@v4 - - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill) - run: | - set -eux - echo "=== container landscape (diagnostic for filter scoping) ===" - docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' - echo "=== end landscape ===" - PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" - PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") - RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") - test -n "$PG_IP" && test -n "$RD_IP" - export DB_HOST="$PG_IP" - export CELERY_BROKER_URL="redis://$RD_IP:6379/0" - export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - for i in $(seq 1 60); do - (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break - sleep 2 - done - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - alembic upgrade head - pytest tests/ -v -m integration --durations=15 \ - --ignore-glob='tests/test_api_*.py' \ - --ignore-glob='tests/test_importer*.py' \ - --ignore-glob='tests/test_import_*.py' \ - --ignore-glob='tests/test_migration_*.py' \ - --ignore-glob='tests/test_phash_*.py' \ - --ignore-glob='tests/test_sidecar_*.py' \ - --ignore-glob='tests/test_scan_*.py' \ - --ignore-glob='tests/test_archive_extractor.py' \ - --ignore-glob='tests/test_backfill_phash.py' + pytest tests/ -v -m integration --durations=15 diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index b82aacf..9785cd7 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -19,7 +19,7 @@ from __future__ import annotations import hashlib from quart import Blueprint, jsonify, request -from sqlalchemy import select +from sqlalchemy import select, text from ..extensions import get_session from ..models import Artist @@ -222,3 +222,49 @@ async def tags_purge_legacy(): lambda sync_sess: purge_legacy_tags(sync_sess, dry_run=dry_run) ) return jsonify(result) + + +@admin_bp.route("/maintenance/db-stats", methods=["GET"]) +async def db_stats(): + """Per-table bloat readout (pg_stat_user_tables) for the high-churn tables + so the operator can see when a VACUUM is worth running.""" + from ..tasks.maintenance import VACUUM_TABLES + + wanted = set(VACUUM_TABLES) + async with get_session() as session: + rows = (await session.execute(text( + "SELECT relname, n_live_tup, n_dead_tup, last_vacuum, " + "last_autovacuum, last_analyze FROM pg_stat_user_tables" + ))).all() + + def _iso(v): + return v.isoformat() if v is not None else None + + out = [] + for r in rows: + if r.relname not in wanted: + continue + live = r.n_live_tup or 0 + dead = r.n_dead_tup or 0 + total = live + dead + out.append({ + "table": r.relname, + "live": live, + "dead": dead, + "dead_pct": round(100 * dead / total, 1) if total else 0.0, + "last_vacuum": _iso(r.last_vacuum), + "last_autovacuum": _iso(r.last_autovacuum), + "last_analyze": _iso(r.last_analyze), + }) + out.sort(key=lambda t: t["dead"], reverse=True) + return jsonify({"tables": out}) + + +@admin_bp.route("/maintenance/vacuum", methods=["POST"]) +async def trigger_vacuum(): + """Operator-triggered VACUUM (ANALYZE) over the high-churn tables — the + same maintenance-queue task the weekly Beat schedule runs.""" + from ..tasks.maintenance import vacuum_analyze + + vacuum_analyze.delay() + return jsonify({"status": "queued"}), 202 diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 1533532..807dd77 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -1,4 +1,6 @@ -"""Gallery API: cursor scroll, timeline, jump, image detail.""" +"""Gallery API: cursor scroll, timeline, jump, image detail, facets.""" + +from datetime import UTC, datetime, timedelta from quart import Blueprint, jsonify, request @@ -8,11 +10,24 @@ from ..services.gallery_service import GalleryService gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") +def _parse_date(raw): + """Parse a YYYY-MM-DD query value to a UTC midnight datetime, or None. + Raises ValueError (→ 400) on a malformed value.""" + if not raw: + return None + return datetime.strptime(raw, "%Y-%m-%d").replace(tzinfo=UTC) + + def _parse_filters(): - """Parse the composable gallery filters from query args. Raises - ValueError (→ 400) on malformed ids. `tag_id` accepts a single id or a - comma-separated list (AND); `media` is image|video; `sort` is - newest|oldest.""" + """Parse the composable gallery filters from query args, returning + ``(filters_dict, sort)``. Raises ValueError (→ 400) on malformed ids/dates. + + `tag_id` accepts a single id or a comma-separated list (AND); `media` is + image|video; `sort` is newest|oldest; `platform` selects one platform + (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are boolean + flags; `date_from`/`date_to` are inclusive calendar-day bounds (date_to is + widened by a day so the whole day is covered by the service's half-open + `< date_to`).""" tag_raw = request.args.get("tag_id") tag_ids = ( [int(x) for x in tag_raw.split(",") if x.strip()] if tag_raw else None @@ -25,7 +40,20 @@ def _parse_filters(): media_type = media if media in ("image", "video") else None sort = request.args.get("sort") sort = sort if sort in ("newest", "oldest") else "newest" - return tag_ids, post_id, artist_id, media_type, sort + platform = request.args.get("platform") or None + untagged = request.args.get("untagged") in ("1", "true", "yes") + no_artist = request.args.get("no_artist") in ("1", "true", "yes") + date_from = _parse_date(request.args.get("date_from")) + date_to = _parse_date(request.args.get("date_to")) + if date_to is not None: + date_to += timedelta(days=1) # inclusive of the date_to calendar day + filters = { + "tag_ids": tag_ids, "post_id": post_id, "artist_id": artist_id, + "media_type": media_type, "platform": platform, + "untagged": untagged, "no_artist": no_artist, + "date_from": date_from, "date_to": date_to, + } + return filters, sort @gallery_bp.route("/scroll", methods=["GET"]) @@ -33,7 +61,7 @@ async def scroll(): cursor = request.args.get("cursor") or None try: limit = int(request.args.get("limit", "50")) - tag_ids, post_id, artist_id, media_type, sort = _parse_filters() + filters, sort = _parse_filters() except ValueError: return jsonify({"error": "invalid filter or limit parameter"}), 400 @@ -41,9 +69,7 @@ async def scroll(): svc = GalleryService(session) try: page = await svc.scroll( - cursor=cursor, limit=limit, tag_ids=tag_ids, - post_id=post_id, artist_id=artist_id, - media_type=media_type, sort=sort, + cursor=cursor, limit=limit, sort=sort, **filters, ) except ValueError as exc: return jsonify({"error": str(exc)}), 400 @@ -75,16 +101,13 @@ async def scroll(): @gallery_bp.route("/timeline", methods=["GET"]) async def timeline(): try: - tag_ids, post_id, artist_id, media_type, _sort = _parse_filters() + filters, _sort = _parse_filters() except ValueError: return jsonify({"error": "invalid filter parameter"}), 400 async with get_session() as session: svc = GalleryService(session) try: - buckets = await svc.timeline( - tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, - media_type=media_type, - ) + buckets = await svc.timeline(**filters) except ValueError as exc: return jsonify({"error": str(exc)}), 400 return jsonify( @@ -92,21 +115,43 @@ async def timeline(): ) +@gallery_bp.route("/facets", methods=["GET"]) +async def facets(): + try: + filters, _sort = _parse_filters() + except ValueError: + return jsonify({"error": "invalid filter parameter"}), 400 + async with get_session() as session: + svc = GalleryService(session) + try: + f = await svc.facets(**filters) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify( + { + "total": f.total, + "platforms": f.platforms, + "untagged": f.untagged, + "no_artist": f.no_artist, + "date_min": f.date_min.isoformat() if f.date_min else None, + "date_max": f.date_max.isoformat() if f.date_max else None, + } + ) + + @gallery_bp.route("/jump", methods=["GET"]) async def jump(): try: year = int(request.args["year"]) month = int(request.args["month"]) - tag_ids, post_id, artist_id, media_type, sort = _parse_filters() + filters, sort = _parse_filters() except (KeyError, ValueError): return jsonify({"error": "year and month query params required"}), 400 async with get_session() as session: svc = GalleryService(session) try: cursor = await svc.jump_cursor( - year=year, month=month, tag_ids=tag_ids, - post_id=post_id, artist_id=artist_id, - media_type=media_type, sort=sort, + year=year, month=month, sort=sort, **filters, ) except ValueError as exc: return jsonify({"error": str(exc)}), 400 diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index eca48e8..6620a6f 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -97,6 +97,10 @@ def make_celery() -> Celery: "task": "backend.app.tasks.maintenance.prune_task_runs", "schedule": 86400.0, # daily }, + "vacuum-analyze": { + "task": "backend.app.tasks.maintenance.vacuum_analyze", + "schedule": 604800.0, # weekly — reclaim dead-tuple bloat + refresh stats + }, "fc3h-backup-db-nightly": { "task": "backend.app.tasks.backup.backup_db_nightly", "schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 7690875..b5060f3 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -18,15 +18,22 @@ import base64 from dataclasses import dataclass from datetime import datetime -from sqlalchemy import Select, and_, exists, func, or_, select +from sqlalchemy import Select, and_, distinct, exists, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import aliased -from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag +from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag from ..models.tag import image_tag CURSOR_SEPARATOR = "|" +# Reserved `platform` filter value selecting images with NO platformed +# provenance (filesystem imports). Returned by facets() as a null-valued +# bucket; the frontend maps that null back to this sentinel in the URL so the +# bucket is selectable. Underscore-wrapped so it can't collide with a real +# gallery-dl platform name (patreon/pixiv/...). +UNSOURCED_PLATFORM = "__unsourced__" + def encode_cursor(effective_date: datetime, image_id: int) -> str: raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}" @@ -92,6 +99,16 @@ class TimelineBucket: count: int +@dataclass(frozen=True) +class GalleryFacets: + total: int # images matching the FULL active filter + platforms: list[dict] # [{"value": str|None, "count": int}], null = unsourced + untagged: int # how many the Untagged flag would isolate + no_artist: int # how many the No-artist flag would isolate + date_min: datetime | None + date_max: datetime | None + + def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str: """Return the URL to fetch a thumbnail. @@ -128,15 +145,27 @@ def _require_single_filter(tag_ids, post_id, artist_id) -> None: ) -def _apply_scope(stmt, *, tag_ids, post_id, artist_id, media_type): - """Apply the composable gallery filters to a statement already joined - to Post via _outer_join_primary_post. +def _apply_scope( + stmt, *, tag_ids, post_id, artist_id, media_type, + platform=None, untagged=False, no_artist=False, + date_from=None, date_to=None, +): + """Apply the composable gallery filters to a statement. - - tag_ids: image must carry ALL of them — one correlated EXISTS per tag - (AND), which avoids the row-multiplication a multi-join would cause. + All clauses are correlated EXISTS / scalar predicates on ImageRecord, so + they AND together without row-multiplication and don't require any join to + be present on `stmt` (the artist/platform paths alias Post/Source inside + their own EXISTS). + + - tag_ids: image must carry ALL of them — one correlated EXISTS per tag. - post_id / artist_id: provenance EXISTS (post_id is exclusive, guarded by _require_single_filter). - media_type: 'image' | 'video' narrows by mime prefix. + - platform: EXISTS a provenance→source with that platform; the + UNSOURCED_PLATFORM sentinel inverts it (NO platformed provenance). + - untagged: NOT EXISTS any image_tag row. + - no_artist: ImageRecord.artist_id IS NULL. + - date_from / date_to: half-open [from, to) bounds on effective_date. """ for tid in tag_ids or []: stmt = stmt.where( @@ -152,9 +181,39 @@ def _apply_scope(stmt, *, tag_ids, post_id, artist_id, media_type): stmt = stmt.where(ImageRecord.mime.like("image/%")) elif media_type == "video": stmt = stmt.where(ImageRecord.mime.like("video/%")) + if platform is not None: + stmt = stmt.where(_platform_clause(platform)) + if untagged: + stmt = stmt.where( + ~exists().where(image_tag.c.image_record_id == ImageRecord.id) + ) + if no_artist: + stmt = stmt.where(ImageRecord.artist_id.is_(None)) + eff = _effective_date_col() + if date_from is not None: + stmt = stmt.where(eff >= date_from) + if date_to is not None: + stmt = stmt.where(eff < date_to) return stmt +def _platform_clause(platform): + """Correlated EXISTS on a provenance row whose Source carries `platform`. + The UNSOURCED_PLATFORM sentinel inverts to NOT EXISTS(any sourced + provenance) — i.e. filesystem-imported content with no platform.""" + src = aliased(Source) + if platform == UNSOURCED_PLATFORM: + return ~exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.source_id == src.id, + ) + return exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.source_id == src.id, + src.platform == platform, + ) + + def _provenance_clause(post_id, artist_id): """Correlated EXISTS clause (NOT a join) so an image with multiple matching provenance rows is returned exactly once and the @@ -215,6 +274,11 @@ class GalleryService: artist_id: int | None = None, media_type: str | None = None, sort: str = "newest", + platform: str | None = None, + untagged: bool = False, + no_artist: bool = False, + date_from: datetime | None = None, + date_to: datetime | None = None, ) -> GalleryPage: if limit < 1 or limit > 200: raise ValueError("limit must be between 1 and 200") @@ -226,6 +290,8 @@ class GalleryService: stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, media_type=media_type, + platform=platform, untagged=untagged, no_artist=no_artist, + date_from=date_from, date_to=date_to, ) descending = sort != "oldest" @@ -286,6 +352,11 @@ class GalleryService: post_id: int | None = None, artist_id: int | None = None, media_type: str | None = None, + platform: str | None = None, + untagged: bool = False, + no_artist: bool = False, + date_from: datetime | None = None, + date_to: datetime | None = None, ) -> list[TimelineBucket]: eff = _effective_date_col() year_col = func.date_part("year", eff).label("yr") @@ -298,6 +369,8 @@ class GalleryService: stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, media_type=media_type, + platform=platform, untagged=untagged, no_artist=no_artist, + date_from=date_from, date_to=date_to, ) stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc()) rows = (await self.session.execute(stmt)).all() @@ -307,6 +380,9 @@ class GalleryService: self, year: int, month: int, tag_ids: list[int] | None = None, post_id: int | None = None, artist_id: int | None = None, media_type: str | None = None, sort: str = "newest", + platform: str | None = None, untagged: bool = False, + no_artist: bool = False, date_from: datetime | None = None, + date_to: datetime | None = None, ) -> str | None: """Returns a cursor that, when passed to scroll() with the same sort, positions at the first image of the given year-month. None if the @@ -324,6 +400,8 @@ class GalleryService: stmt = _apply_scope( stmt, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, media_type=media_type, + platform=platform, untagged=untagged, no_artist=no_artist, + date_from=date_from, date_to=date_to, ) descending = sort != "oldest" if descending: @@ -339,6 +417,94 @@ class GalleryService: boundary = record.id + 1 if descending else record.id - 1 return encode_cursor(eff_date, boundary) + async def facets( + self, *, tag_ids: list[int] | None = None, + post_id: int | None = None, artist_id: int | None = None, + media_type: str | None = None, platform: str | None = None, + untagged: bool = False, no_artist: bool = False, + date_from: datetime | None = None, date_to: datetime | None = None, + ) -> GalleryFacets: + """Live facet counts scoped to the current filter. Each facet GROUP is + computed with all OTHER active filters applied but its OWN selection + ignored ("minus-self"), so sibling options stay visible/switchable. + No outer join is needed — every clause is a correlated EXISTS or a + column predicate on ImageRecord. + """ + _require_single_filter(tag_ids, post_id, artist_id) + common = { + "tag_ids": tag_ids, "post_id": post_id, + "artist_id": artist_id, "media_type": media_type, + } + + # total — the full active filter (the headline result count). + total = (await self.session.execute( + _apply_scope( + select(func.count(ImageRecord.id)), **common, + platform=platform, untagged=untagged, no_artist=no_artist, + date_from=date_from, date_to=date_to, + ) + )).scalar_one() + + # platforms — scope minus the platform selection. Inner-join + # provenance→source and COUNT(DISTINCT image) per platform (a + # cross-posted image counts under each of its platforms). + plat_scope = { + **common, "untagged": untagged, "no_artist": no_artist, + "date_from": date_from, "date_to": date_to, + } + src = aliased(Source) + plat_stmt = ( + select(src.platform, func.count(distinct(ImageRecord.id))) + .select_from(ImageRecord) + .join(ImageProvenance, ImageProvenance.image_record_id == ImageRecord.id) + .join(src, src.id == ImageProvenance.source_id) + ) + plat_stmt = _apply_scope(plat_stmt, **plat_scope).group_by(src.platform) + platforms = [ + {"value": p, "count": c} + for p, c in (await self.session.execute(plat_stmt)).all() + ] + # Unsourced (filesystem) bucket — same minus-platform scope. + unsourced = (await self.session.execute( + _apply_scope( + select(func.count(ImageRecord.id)), **plat_scope, + platform=UNSOURCED_PLATFORM, + ) + )).scalar_one() + if unsourced: + platforms.append({"value": None, "count": unsourced}) + + # curation flags — each minus its OWN flag. + untagged_count = (await self.session.execute( + _apply_scope( + select(func.count(ImageRecord.id)), **common, + platform=platform, no_artist=no_artist, + date_from=date_from, date_to=date_to, untagged=True, + ) + )).scalar_one() + no_artist_count = (await self.session.execute( + _apply_scope( + select(func.count(ImageRecord.id)), **common, + platform=platform, untagged=untagged, + date_from=date_from, date_to=date_to, no_artist=True, + ) + )).scalar_one() + + # date bounds — scope minus the date params (those drive the picker). + eff = _effective_date_col() + dmin, dmax = (await self.session.execute( + _apply_scope( + select(func.min(eff), func.max(eff)), **common, + platform=platform, untagged=untagged, no_artist=no_artist, + ) + )).one() + + return GalleryFacets( + total=total, platforms=platforms, + untagged=untagged_count, no_artist=no_artist_count, + date_min=dmin, date_max=dmax, + ) + async def get_image_with_tags(self, image_id: int) -> dict | None: record = await self.session.get(ImageRecord, image_id) if record is None: diff --git a/backend/app/tasks/_sync_engine.py b/backend/app/tasks/_sync_engine.py index 508e490..eccfb84 100644 --- a/backend/app/tasks/_sync_engine.py +++ b/backend/app/tasks/_sync_engine.py @@ -34,3 +34,10 @@ def sync_session_factory(): ) _SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False) return _SESSIONMAKER + + +def get_sync_engine(): + """The process-wide sync Engine — for raw work that needs a connection + directly (e.g. AUTOCOMMIT VACUUM, which can't run inside a transaction).""" + sync_session_factory() # ensure _ENGINE is initialized + return _ENGINE diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index ada9f3e..2be4a26 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -22,10 +22,27 @@ from ..models import ( TaskRun, ) from ..utils.phash import compute_phash +from ._sync_engine import get_sync_engine from ._sync_engine import sync_session_factory as _sync_session_factory log = logging.getLogger(__name__) +# High-churn tables whose dead-tuple bloat matters: the TABLESAMPLE showcase +# reads physical blocks (bloat slows it directly), and the periodic +# prune/backfill/recovery tasks generate dead tuples faster than autovacuum +# always keeps up with. VACUUM reclaims them; ANALYZE refreshes planner stats. +# Allowlist ONLY — names are interpolated into VACUUM, so they must never come +# from request input. +VACUUM_TABLES = ( + "image_record", + "image_provenance", + "post_attachment", + "download_event", + "task_run", + "import_task", + "import_batch", +) + STUCK_THRESHOLD_MINUTES = 5 # Archive ImportTasks run the per-member pipeline inline for every # member (import_archive_file: soft=30min/hard=35min). The ImportTask @@ -761,3 +778,20 @@ def cleanup_old_download_events() -> int: ) session.commit() return result.rowcount or 0 + + +@celery.task(name="backend.app.tasks.maintenance.vacuum_analyze") +def vacuum_analyze() -> dict: + """Periodic VACUUM (ANALYZE) over the high-churn tables (VACUUM_TABLES) to + reclaim dead-tuple bloat and refresh planner statistics. VACUUM cannot run + inside a transaction block, so it runs on an AUTOCOMMIT connection. + Scheduled weekly; also operator-triggerable from Settings → Maintenance. + """ + engine = get_sync_engine() + done = [] + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + for table in VACUUM_TABLES: + conn.exec_driver_sql(f"VACUUM (ANALYZE) {table}") + done.append(table) + log.info("vacuum_analyze complete: %s", done) + return {"vacuumed": done} diff --git a/frontend/src/components/discovery/MasonryGrid.vue b/frontend/src/components/discovery/MasonryGrid.vue index 436b8e9..475482e 100644 --- a/frontend/src/components/discovery/MasonryGrid.vue +++ b/frontend/src/components/discovery/MasonryGrid.vue @@ -6,7 +6,6 @@ v-for="item in col" :key="item.id" class="fc-masonry__item" :class="{ 'fc-masonry__item--anim': shouldAnimate(item) }" - :style="itemStyle(item)" type="button" @click="$emit('open', item.id)" > @@ -66,12 +65,6 @@ function shouldAnimate(item) { return idx !== undefined && idx >= props.animateFromIndex } -function itemStyle(item) { - if (!shouldAnimate(item)) return {} - const idx = idxById.value.get(item.id) - props.animateFromIndex - return { '--stagger-index': idx } -} - function aspectStyle(item) { const w = Number(item.width) const h = Number(item.height) @@ -117,12 +110,15 @@ useInfiniteScroll(sentinelEl, () => { .fc-masonry__end { text-align: center; padding: 32px 0; } /* Cascade entry: each tile flips up out of a backward tilt and settles - into place, one at a time — more pronounced than a plain fade so the - showcase reads as an "experience" (operator-flagged 2026-05-28). The - `both` fill holds the hidden/tilted 0% state until each tile's staggered - turn; the cubic-bezier overshoots slightly past flat then settles. - Honors prefers-reduced-motion. Tunables: tilt (-28deg), stagger (70ms), - duration (0.6s). */ + into place — more pronounced than a plain fade so the showcase reads as an + "experience" (operator-flagged 2026-05-28). The reveal is paced entirely by + the store, which pushes one fully-decoded item at a time (showcase.js); each + tile therefore animates the instant it mounts, with NO per-index CSS delay — + the old `animation-delay: index×70ms` compounded on top of the insert + cadence and made the cascade drag and desync as it grew (operator-flagged + 2026-06-04). The `both` fill holds the hidden/tilted 0% state until mount; + the cubic-bezier overshoots slightly past flat then settles. Honors + prefers-reduced-motion. Tunables: tilt (-28deg), duration (0.6s). */ @keyframes fc-masonry-item-in { 0% { opacity: 0; @@ -138,7 +134,6 @@ useInfiniteScroll(sentinelEl, () => { transform-origin: center top; backface-visibility: hidden; animation: fc-masonry-item-in 0.6s cubic-bezier(0.34, 1.45, 0.64, 1) both; - animation-delay: calc(var(--stagger-index, 0) * 70ms); } @media (prefers-reduced-motion: reduce) { .fc-masonry__item--anim { diff --git a/frontend/src/components/gallery/GalleryFacetPanel.vue b/frontend/src/components/gallery/GalleryFacetPanel.vue new file mode 100644 index 0000000..bb09916 --- /dev/null +++ b/frontend/src/components/gallery/GalleryFacetPanel.vue @@ -0,0 +1,146 @@ + + + + + diff --git a/frontend/src/components/gallery/GalleryFilterBar.vue b/frontend/src/components/gallery/GalleryFilterBar.vue index 9db956a..a19063f 100644 --- a/frontend/src/components/gallery/GalleryFilterBar.vue +++ b/frontend/src/components/gallery/GalleryFilterBar.vue @@ -1,5 +1,6 @@ diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index e332e1c..f550c77 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -14,6 +14,7 @@ +