diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 45b3571..cb247cf 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -1,7 +1,8 @@ name: CI # CI lanes per FabledRulebook/forgejo.md "CI philosophy": -# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers. +# - lint: ruff only, no dep install — fast-fail for the common lint bounce. +# - backend-lint-and-test: `pytest -m "not integration"`, no service containers. # - frontend-build: vitest unit + vite build. # - integration: pgvector + redis service containers; alembic + `pytest -m integration`. @@ -14,6 +15,20 @@ on: # (single-operator Forgejo repo) so push coverage is complete. jobs: + # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so + # this runs with NO dependency install and surfaces the most common bounce + # class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of + # after the backend job's ~30-60s wheel install. ruff is static analysis, + # so no DB/secret env is needed. + lint: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + - name: Ruff lint + run: ruff check backend/ tests/ alembic/ + backend-lint-and-test: runs-on: python-ci container: @@ -51,9 +66,8 @@ jobs: pip install -r requirements.txt pytest pytest-asyncio fi - - name: Ruff lint - run: ruff check backend/ tests/ alembic/ - + # Ruff moved to the dedicated fast `lint` job above (fails in seconds, + # no dep install). This job is now unit tests only. - name: Pytest (unit only — integration runs in the integration job) run: pytest tests/ -v -m "not integration" diff --git a/alembic/versions/0027_drop_migration_run.py b/alembic/versions/0027_drop_migration_run.py new file mode 100644 index 0000000..454481b --- /dev/null +++ b/alembic/versions/0027_drop_migration_run.py @@ -0,0 +1,50 @@ +"""drop migration_run — one-and-done GS/IR migration tooling removed + +Revision ID: 0027 +Revises: 0026 +Create Date: 2026-05-29 + +The GS/IR migration tooling (services/migrators, /api/migrate, the +run_migration task, LegacyMigrationCard, and the MigrationRun model) was +removed after the migration cutover completed. This drops its now-orphaned +run-log table. Downgrade recreates the table (mirrors the old model) so the +migration is reversible. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +revision: str = "0027" +down_revision: Union[str, None] = "0026" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table("migration_run") + + +def downgrade() -> None: + op.create_table( + "migration_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + op.create_index("ix_migration_run_kind", "migration_run", ["kind"]) + op.create_index("ix_migration_run_status", "migration_run", ["status"]) diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 70efbda..75ace0d 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -33,12 +33,6 @@ def create_app() -> Quart: app = Quart(__name__) app.secret_key = cfg.secret_key - # FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of - # thousands of image_tag_associations). Werkzeug's default form - # memory cap is 500KB; raise both ceilings so the multipart upload - # for /api/migrate/ir_ingest doesn't 413. - app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GB - app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024 # 1 GB for bp in all_blueprints(): app.register_blueprint(bp) diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index b8dc7c4..f97b72a 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -26,7 +26,6 @@ def all_blueprints() -> list[Blueprint]: from .extension import extension_bp from .gallery import gallery_bp from .import_admin import import_admin_bp - from .migrate import migrate_bp from .ml_admin import ml_admin_bp from .platforms import platforms_bp from .posts import posts_bp @@ -54,7 +53,6 @@ def all_blueprints() -> list[Blueprint]: admin_bp, cleanup_bp, import_admin_bp, - migrate_bp, suggestions_bp, allowlist_bp, aliases_bp, diff --git a/backend/app/api/migrate.py b/backend/app/api/migrate.py deleted file mode 100644 index d48e9f4..0000000 --- a/backend/app/api/migrate.py +++ /dev/null @@ -1,107 +0,0 @@ -"""FC-5: /api/migrate — trigger and poll migration runs. - -Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an -`export_file` field. All other kinds accept JSON. Backup + rollback -were retired in FC-3h (2026-05-24); use /api/system/backup/* instead. -""" - -import json - -from quart import Blueprint, jsonify, request -from sqlalchemy import select - -from ..extensions import get_session -from ..models import MigrationRun -from ..tasks.migration import run_migration -from ._responses import error_response as _bad - -migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate") - -# 'backup' + 'rollback' retired 2026-05-24 (FC-3h); see /api/system/backup/*. -_VALID_KINDS = frozenset({ - "gs_ingest", "ir_ingest", "tag_apply", - "ml_queue", "verify", "cleanup", -}) -_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"}) - - -def _run_to_dict(run: MigrationRun) -> dict: - return { - "id": run.id, - "kind": run.kind, - "status": run.status, - "dry_run": run.dry_run, - "started_at": run.started_at.isoformat(), - "finished_at": run.finished_at.isoformat() if run.finished_at else None, - "counts": run.counts or {}, - "error": run.error, - "metadata": run.metadata_ or {}, - } - - -@migrate_bp.route("/", methods=["POST"]) -async def create_run(kind: str): - if kind not in _VALID_KINDS: - return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}") - - # Ingest kinds accept multipart/form-data; everything else takes JSON. - if kind in _INGEST_KINDS: - form = await request.form - files = await request.files - if "export_file" not in files: - return _bad("missing_export_file", detail="multipart export_file required") - export_file = files["export_file"] - try: - raw = export_file.read() - data = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - return _bad("invalid_export_file", detail=str(exc)) - dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes") - params: dict = {"data": data, "dry_run": dry_run} - else: - body = await request.get_json() - if body is None: - body = {} - if not isinstance(body, dict): - return _bad("invalid_body") - dry_run = bool(body.get("dry_run", False)) - params = dict(body) - - async with get_session() as session: - run = MigrationRun(kind=kind, status="pending", dry_run=dry_run) - session.add(run) - await session.commit() - await session.refresh(run) - run_id = run.id - - run_migration.delay(run_id, kind, params) - return jsonify({"run_id": run_id, "status": "pending"}), 202 - - -@migrate_bp.route("/runs/", methods=["GET"]) -async def get_run(run_id: int): - async with get_session() as session: - run = (await session.execute( - select(MigrationRun).where(MigrationRun.id == run_id) - )).scalar_one_or_none() - if run is None: - return _bad("not_found", status=404) - return jsonify(_run_to_dict(run)) - - -@migrate_bp.route("/runs", methods=["GET"]) -async def list_runs(): - try: - limit = int(request.args.get("limit", "10")) - except ValueError: - return _bad("invalid_limit") - if limit < 1 or limit > 100: - return _bad("invalid_limit") - - async with get_session() as session: - rows = (await session.execute( - select(MigrationRun) - .order_by(MigrationRun.id.desc()) - .limit(limit) - )).scalars().all() - return jsonify([_run_to_dict(r) for r in rows]) diff --git a/backend/app/api/posts.py b/backend/app/api/posts.py index d96e523..9a3edb2 100644 --- a/backend/app/api/posts.py +++ b/backend/app/api/posts.py @@ -18,6 +18,8 @@ async def list_posts(): artist_id_raw = args.get("artist_id") platform = args.get("platform") or None limit_raw = args.get("limit", "24") + direction = args.get("direction", "older") + around_raw = args.get("around") try: limit = int(limit_raw) @@ -26,6 +28,16 @@ async def list_posts(): if limit < 1 or limit > 100: return _bad("invalid_limit", detail="limit must be between 1 and 100") + if direction not in ("older", "newer"): + return _bad("invalid_direction", detail="direction must be 'older' or 'newer'") + + around_id = None + if around_raw is not None: + try: + around_id = int(around_raw) + except ValueError: + return _bad("invalid_around", detail="around must be an integer post id") + artist_id = None if artist_id_raw is not None: try: @@ -40,11 +52,20 @@ async def list_posts(): ) async with get_session() as session: - try: - page = await PostFeedService(session).scroll( - cursor=cursor, artist_id=artist_id, + svc = PostFeedService(session) + if around_id is not None: + result = await svc.around( + post_id=around_id, artist_id=artist_id, platform=platform, limit=limit, ) + if result is None: + return _bad("not_found", status=404, detail=f"post id={around_id}") + return jsonify(result) + try: + page = await svc.scroll( + cursor=cursor, artist_id=artist_id, + platform=platform, limit=limit, direction=direction, + ) except ValueError as exc: # Service raises ValueError for malformed cursors only; # limit bounds are validated above. diff --git a/backend/app/api/system_activity.py b/backend/app/api/system_activity.py index 480b8c9..0d0600c 100644 --- a/backend/app/api/system_activity.py +++ b/backend/app/api/system_activity.py @@ -20,6 +20,7 @@ from sqlalchemy import desc, func, select from ..config import get_config from ..extensions import get_session from ..models import TaskRun +from ..services.scheduler_service import scheduler_status system_activity_bp = Blueprint( "system_activity", __name__, url_prefix="/api/system/activity", @@ -81,17 +82,22 @@ def _read_workers_sync() -> dict: } +async def _queues_cached() -> dict: + """Per-queue Redis LLEN, cached 2s. Shared by /queues and /summary.""" + now = time.time() + if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL: + _QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync) + _QUEUE_CACHE["ts"] = now + return _QUEUE_CACHE["data"] + + @system_activity_bp.route("/queues", methods=["GET"]) async def get_queues(): """Per-queue Redis LLEN. Cached 2s. Response: {queues: {name: depth_or_null}, fetched_at: iso8601} """ - now = time.time() - if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL: - _QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync) - _QUEUE_CACHE["ts"] = now - return jsonify(_QUEUE_CACHE["data"]) + return jsonify(await _queues_cached()) @system_activity_bp.route("/workers", methods=["GET"]) @@ -107,6 +113,35 @@ async def get_workers(): return jsonify(_WORKER_CACHE["data"]) +@system_activity_bp.route("/summary", methods=["GET"]) +async def get_summary(): + """One-call rollup for the always-on TopNav pipeline indicator: + scheduler health, per-queue pending depths, currently-running count, and + recent (24h) failure count. Cheap — cached queue LLENs + two TaskRun + counts — so it's safe to poll app-wide.""" + queues_data = await _queues_cached() + depths = queues_data.get("queues", {}) + queued_total = sum(v for v in depths.values() if isinstance(v, int)) + since = datetime.now(UTC) - timedelta(hours=24) + async with get_session() as session: + scheduler = await scheduler_status(session) + running = (await session.execute( + select(func.count(TaskRun.id)).where(TaskRun.status == "running") + )).scalar_one() + failing = (await session.execute( + select(func.count(TaskRun.id)) + .where(TaskRun.status.in_(["error", "timeout"])) + .where(TaskRun.finished_at >= since) + )).scalar_one() + return jsonify({ + "scheduler": scheduler, + "queues": depths, + "queued_total": queued_total, + "running": int(running), + "failing": int(failing), + }) + + @system_activity_bp.route("/runs", methods=["GET"]) async def list_runs(): """Paginated task_run history. Query params: diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 3068ca2..eba1a64 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -28,7 +28,6 @@ def make_celery() -> Celery: "backend.app.tasks.import_file", "backend.app.tasks.thumbnail", "backend.app.tasks.maintenance", - "backend.app.tasks.migration", "backend.app.tasks.ml", "backend.app.tasks.download", "backend.app.tasks.backup", @@ -45,7 +44,6 @@ def make_celery() -> Celery: "backend.app.tasks.download.*": {"queue": "download"}, "backend.app.tasks.scan.*": {"queue": "scan"}, "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, - "backend.app.tasks.migration.*": {"queue": "maintenance"}, "backend.app.tasks.backup.*": {"queue": "maintenance"}, "backend.app.tasks.admin.*": {"queue": "maintenance"}, "backend.app.tasks.library_audit.*": {"queue": "maintenance"}, @@ -87,6 +85,10 @@ def make_celery() -> Celery: "task": "backend.app.tasks.maintenance.cleanup_old_download_events", "schedule": 86400.0, # daily }, + "recover-stalled-download-events": { + "task": "backend.app.tasks.maintenance.recover_stalled_download_events", + "schedule": 300.0, # every 5 min, matches recover-interrupted-tasks + }, "recover-stalled-task-runs": { "task": "backend.app.tasks.maintenance.recover_stalled_task_runs", "schedule": 300.0, # every 5 min, matches recover-interrupted-tasks diff --git a/backend/app/celery_signals.py b/backend/app/celery_signals.py index 307d5ea..37bd482 100644 --- a/backend/app/celery_signals.py +++ b/backend/app/celery_signals.py @@ -66,10 +66,7 @@ def _queue_for(task) -> str: return "download" if name.startswith("backend.app.tasks.scan."): return "scan" - if name.startswith(( - "backend.app.tasks.maintenance.", - "backend.app.tasks.migration.", - )): + if name.startswith("backend.app.tasks.maintenance."): return "maintenance" return "default" diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 00a7c80..eec9da0 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -12,7 +12,6 @@ from .import_batch import ImportBatch from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun -from .migration_run import MigrationRun from .ml_settings import MLSettings from .post import Post from .post_attachment import PostAttachment @@ -46,7 +45,6 @@ __all__ = [ "ImportSettings", "LibraryAuditRun", "MLSettings", - "MigrationRun", "TagAlias", "TagAllowlist", "TagReferenceEmbedding", diff --git a/backend/app/models/migration_run.py b/backend/app/models/migration_run.py deleted file mode 100644 index bdb5fed..0000000 --- a/backend/app/models/migration_run.py +++ /dev/null @@ -1,37 +0,0 @@ -"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc). - -kind/status are String(32) not Postgres ENUM so adding kinds later -doesn't need a schema migration. The API layer validates values. -""" - -from datetime import datetime - -import sqlalchemy as sa -from sqlalchemy import Boolean, DateTime, Integer, String, Text, func -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.orm import Mapped, mapped_column - -from .base import Base - - -class MigrationRun(Base): - __tablename__ = "migration_run" - - id: Mapped[int] = mapped_column(Integer, primary_key=True) - kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True) - status: Mapped[str] = mapped_column(String(32), nullable=False, index=True) - dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - started_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now(), - ) - finished_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, - ) - counts: Mapped[dict] = mapped_column( - JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"), - ) - error: Mapped[str | None] = mapped_column(Text, nullable=True) - metadata_: Mapped[dict] = mapped_column( - "metadata", JSONB, nullable=False, default=dict, - server_default=sa.text("'{}'::jsonb"), - ) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index dba9323..0e817c4 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -6,9 +6,8 @@ HTTP handlers (small ops) and from Celery tasks in backend.app.tasks.admin (long ops). This module is the PERMANENT home of artist-cascade + image-unlink -logic. The legacy copy at backend/app/services/migrators/cleanup.py -stays in place until FC-3j; FC-3j will replace its body with thin -re-exports from this module and then delete the wrapper. +logic. (The legacy migrators/cleanup.py copy was removed with the rest of +the one-and-done GS/IR migration tooling.) """ from __future__ import annotations diff --git a/backend/app/services/migrators/__init__.py b/backend/app/services/migrators/__init__.py deleted file mode 100644 index b7c0908..0000000 --- a/backend/app/services/migrators/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""FC-5 migration tooling. - -One module per concern (gs/ir/overlap/ml_queue/verify/cleanup). -Each migrator returns a counts dict; the run_migration task wires -that dict into MigrationRun.counts so the UI polling shows progress. - -backup + rollback were retired in FC-3h (2026-05-24); first-class -backup lives at backend/app/services/backup_service.py and exposes -its own /api/system/backup/* surface. -""" diff --git a/backend/app/services/migrators/cleanup.py b/backend/app/services/migrators/cleanup.py deleted file mode 100644 index 9f8da2c..0000000 --- a/backend/app/services/migrators/cleanup.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Targeted cleanup migrator: delete every image attributed to one Artist. - -Built for the IR-migration rescue case where the filesystem scan derived -a bogus 'imagerepo' artist from a mismatched bind-mount layout. Every -image attributed to that artist (40k+ rows) needs to be removed — DB -rows, original files under `/images//...`, and thumbnails under -`/images/thumbs/...` — before the operator remounts and re-scans. - -CASCADE handles image_tag, image_provenance, series_page, and -tag_suggestion_rejection child rows; import_task.result_image_id is -SET NULL by FK. We also delete ImportTask rows whose source_path starts -with the (still-existing) IR scan prefix so the next scan isn't fooled -by them. -""" -from __future__ import annotations - -import logging -from pathlib import Path - -from sqlalchemy import delete, func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import Artist, ImageRecord, ImportBatch, ImportTask - -log = logging.getLogger(__name__) - -_BATCH_SIZE = 500 - - -def _zero_counts() -> dict: - return { - "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, - "files_copied": 0, "bytes_copied": 0, "conflicts": 0, - } - - -def _thumb_path(images_root: Path, sha256_hex: str) -> tuple[Path, Path]: - """Return both possible thumbnail paths (.jpg and .png). We try both - because the extension is chosen at generate-time based on the source - image's mode (alpha → .png, otherwise → .jpg).""" - bucket = sha256_hex[:3] - base = images_root / "thumbs" / bucket / sha256_hex - return base.with_suffix(".jpg"), base.with_suffix(".png") - - -def _delete_file(path: Path) -> bool: - """Best-effort unlink; True if the file was actually removed.""" - try: - path.unlink(missing_ok=True) - return True - except OSError as exc: - log.warning("cleanup: failed to unlink %s: %s", path, exc) - return False - - -async def cleanup_artist_async( - db: AsyncSession, - *, - slug: str, - images_root: Path | None = None, - dry_run: bool = False, - source_path_prefix: str | None = None, -) -> dict: - """Delete every image attributed to the Artist with this slug, - along with the artist row itself and any associated import tasks. - - Args: - slug: artist.slug to target (e.g. 'imagerepo'). - images_root: defaults to /images. - dry_run: skip filesystem + DB writes; still walk rows for counts. - source_path_prefix: if set, ImportTask rows whose source_path - starts with this string are deleted too (use the IR scan - mount prefix, e.g. '/import/imagerepo'). - """ - root = images_root if images_root is not None else Path("/images") - - artist = (await db.execute( - select(Artist).where(Artist.slug == slug) - )).scalar_one_or_none() - if artist is None: - raise ValueError(f"no Artist with slug={slug!r}") - - artist_id = artist.id - artist_name = artist.name - - total_images = (await db.execute( - select(func.count(ImageRecord.id)).where(ImageRecord.artist_id == artist_id) - )).scalar_one() - - counts = _zero_counts() - files_deleted = 0 - thumbs_deleted = 0 - images_deleted = 0 - - # Batched delete loop. CASCADE handles image_tag, image_provenance, - # series_page, tag_suggestion_rejection. import_task.result_image_id - # is SET NULL by FK. - while True: - rows = (await db.execute( - select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256) - .where(ImageRecord.artist_id == artist_id) - .limit(_BATCH_SIZE) - )).all() - if not rows: - break - - ids = [r.id for r in rows] - counts["rows_processed"] += len(ids) - - if not dry_run: - for r in rows: - if r.path: - if _delete_file(Path(r.path)): - files_deleted += 1 - if r.sha256: - jpg, png = _thumb_path(root, r.sha256) - if _delete_file(jpg): - thumbs_deleted += 1 - if _delete_file(png): - thumbs_deleted += 1 - - await db.execute( - delete(ImageRecord).where(ImageRecord.id.in_(ids)) - ) - await db.commit() - - images_deleted += len(ids) - - if dry_run: - # Nothing was actually deleted from the DB; bail after one - # pass so we don't loop forever. - break - - import_tasks_deleted = 0 - if source_path_prefix and not dry_run: - # Delete ImportTask rows whose source_path is under the bad mount - # prefix. These are mostly orphaned now (result_image_id was set - # NULL by CASCADE) but their presence still blocks the - # idempotency check in scan_directory if the operator remounts - # the same prefix. - like_pattern = source_path_prefix.rstrip("/") + "/%" - result = await db.execute( - delete(ImportTask).where(ImportTask.source_path.like(like_pattern)) - ) - import_tasks_deleted = result.rowcount or 0 - await db.commit() - - # Sweep ImportBatch rows that are now empty. - empty_batches_deleted = 0 - if not dry_run: - empty_batch_ids = (await db.execute( - select(ImportBatch.id).where( - ~select(ImportTask.id) - .where(ImportTask.batch_id == ImportBatch.id) - .exists() - ) - )).scalars().all() - if empty_batch_ids: - result = await db.execute( - delete(ImportBatch).where(ImportBatch.id.in_(empty_batch_ids)) - ) - empty_batches_deleted = result.rowcount or 0 - await db.commit() - - # Finally, the artist row. - if not dry_run: - await db.execute(delete(Artist).where(Artist.id == artist_id)) - await db.commit() - - return { - "counts": counts, - "artist": {"id": artist_id, "name": artist_name, "slug": slug}, - "summary": { - "images_targeted": total_images, - "images_deleted": images_deleted, - "files_deleted": files_deleted, - "thumbs_deleted": thumbs_deleted, - "import_tasks_deleted": import_tasks_deleted, - "empty_batches_deleted": empty_batches_deleted, - "dry_run": dry_run, - }, - } diff --git a/backend/app/services/migrators/gs_ingest.py b/backend/app/services/migrators/gs_ingest.py deleted file mode 100644 index 18b832a..0000000 --- a/backend/app/services/migrators/gs_ingest.py +++ /dev/null @@ -1,126 +0,0 @@ -"""GallerySubscriber export → FabledCurator ingest. - -Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection -to GS). Creates Artist (from subscriptions) + Source (nested under each -subscription) + Credential (re-encrypted with FC's key). Idempotent on -natural keys: Artist.slug, (artist_id, platform, url), Credential.platform. - -Credentials arrive plaintext in the export — GS's export script -decrypts using GS's Fernet key in GS's own process. FC re-encrypts -with FC's CredentialCrypto. -""" -from __future__ import annotations - -import json - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import Artist, Credential, Source -from ...utils.slug import slugify -from ..credential_crypto import CredentialCrypto - - -def _zero_counts() -> dict: - return { - "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, - "files_copied": 0, "bytes_copied": 0, "conflicts": 0, - } - - -async def migrate_async( - db: AsyncSession, - *, - data: dict, - fc_crypto: CredentialCrypto | None = None, - dry_run: bool = False, -) -> dict: - """Ingest a parsed gallerysubscriber-export-v1.json dict.""" - if data.get("source_app") != "gallerysubscriber": - raise ValueError("export source_app must be 'gallerysubscriber'") - if data.get("schema_version") != 1: - raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") - - counts = _zero_counts() - - # Phase 1: subscriptions → Artist; nested sources within each. - for sub in data.get("subscriptions", []): - counts["rows_processed"] += 1 - slug = slugify(sub["name"]) - artist = (await db.execute( - select(Artist).where(Artist.slug == slug) - )).scalar_one_or_none() - if artist is None: - if dry_run: - counts["rows_inserted"] += 1 - # Continue to nested sources, but they can't link without an artist row. - continue - notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None - artist = Artist( - name=sub["name"], slug=slug, - is_subscription=True, - auto_check=bool(sub.get("enabled", True)), - notes=notes, - ) - db.add(artist) - await db.flush() - counts["rows_inserted"] += 1 - else: - counts["rows_skipped"] += 1 - - # Nested sources under this subscription. - for src in sub.get("sources", []): - counts["rows_processed"] += 1 - existing = (await db.execute( - select(Source).where( - Source.artist_id == artist.id, - Source.platform == src["platform"], - Source.url == src["url"], - ) - )).scalar_one_or_none() - if existing is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - db.add(Source( - artist_id=artist.id, - platform=src["platform"], - url=src["url"], - enabled=bool(src.get("enabled", True)), - check_interval_override=src.get("check_interval"), - config_overrides=src.get("metadata") or {}, - )) - counts["rows_inserted"] += 1 - - # Phase 2: credentials. - for cred in data.get("credentials", []): - counts["rows_processed"] += 1 - existing = (await db.execute( - select(Credential).where(Credential.platform == cred["platform"]) - )).scalar_one_or_none() - if existing is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - if fc_crypto is None: - # Without a crypto helper we can't encrypt — skip rather than - # store plaintext. - counts["rows_skipped"] += 1 - counts["conflicts"] += 1 - continue - encrypted = fc_crypto.encrypt(cred["plaintext"]) - db.add(Credential( - platform=cred["platform"], - credential_type=cred.get("credential_type") or "cookies", - encrypted_blob=encrypted, - expires_at=cred.get("expires_at"), - )) - counts["rows_inserted"] += 1 - - if not dry_run: - await db.commit() - return counts diff --git a/backend/app/services/migrators/ir_ingest.py b/backend/app/services/migrators/ir_ingest.py deleted file mode 100644 index b501583..0000000 --- a/backend/app/services/migrators/ir_ingest.py +++ /dev/null @@ -1,146 +0,0 @@ -"""ImageRepo export → FabledCurator ingest. - -Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR). -Creates Tag rows (skipping artist/post kinds, resolving fandom_name to -FK). Writes the per-image-sha256 artist assignments + tag associations -+ series page assignments to /images/_migration_state/ir_tag_manifest.json -so tag_apply.py can join them to ImageRecord rows AFTER the operator -runs FC's filesystem scan. -""" -from __future__ import annotations - -import json -from pathlib import Path - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import Tag, TagKind - -_SKIP_KINDS = frozenset({"artist", "post"}) -_MIGRATION_STATE_DIRNAME = "_migration_state" -_IR_MANIFEST_FILENAME = "ir_tag_manifest.json" - - -def _zero_counts() -> dict: - return { - "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, - "files_copied": 0, "bytes_copied": 0, "conflicts": 0, - } - - -def manifest_path(images_root: Path | None = None) -> Path: - root = images_root if images_root is not None else Path("/images") - p = root / _MIGRATION_STATE_DIRNAME - p.mkdir(parents=True, exist_ok=True) - return p / _IR_MANIFEST_FILENAME - - -async def _resolve_fandom_id( - db: AsyncSession, fandom_name: str | None, dry_run: bool, -) -> int | None: - """Find-or-create a fandom-kind Tag by name.""" - if not fandom_name: - return None - existing = (await db.execute( - select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom") - )).scalar_one_or_none() - if existing is not None: - return existing.id - if dry_run: - return None - t = Tag(name=fandom_name, kind=TagKind.fandom) - db.add(t) - await db.flush() - return t.id - - -async def migrate_async( - db: AsyncSession, - *, - data: dict, - images_root: Path | None = None, - dry_run: bool = False, -) -> dict: - """Ingest a parsed imagerepo-export-v1.json dict. - - Creates Tag rows + writes the IR tag manifest file. Tag-to-image - binding happens later in tag_apply.py (after FC's filesystem scan - populates image_record.sha256 → id). - """ - if data.get("source_app") != "imagerepo": - raise ValueError("export source_app must be 'imagerepo'") - if data.get("schema_version") not in (1, 2): - raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") - - counts = _zero_counts() - - # Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id). - # First pass: create all fandom-kind tags so they're available for FK resolution. - for tag in data.get("tags", []): - kind = tag.get("kind") or "general" - if kind != "fandom": - continue - counts["rows_processed"] += 1 - existing = (await db.execute( - select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom") - )).scalar_one_or_none() - if existing is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - db.add(Tag(name=tag["name"], kind=TagKind.fandom)) - counts["rows_inserted"] += 1 - if not dry_run: - await db.flush() - - # Second pass: every other kind. - for tag in data.get("tags", []): - kind_str = tag.get("kind") or "general" - if kind_str in _SKIP_KINDS: - counts["rows_skipped"] += 1 - continue - if kind_str == "fandom": - continue # handled above - counts["rows_processed"] += 1 - try: - kind = TagKind(kind_str) - except ValueError: - kind = TagKind.general - fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run) - existing = (await db.execute( - select(Tag).where(Tag.name == tag["name"], Tag.kind == kind) - )).scalar_one_or_none() - if existing is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id)) - counts["rows_inserted"] += 1 - - if not dry_run: - await db.commit() - - # Phase 2: write the per-image manifest for tag_apply.py to consume later. - # schema_version 2 (added 2026-05-24) carries `image_posts` for - # Post + Source + ImageProvenance restore; schema 1 manifests - # without it stay valid (tag_apply treats the missing field as []). - manifest = { - "schema_version": data.get("schema_version", 1), - "image_artist_assignments": data.get("image_artist_assignments", []), - "image_tag_associations": data.get("image_tag_associations", []), - "series_pages": data.get("series_pages", []), - "image_posts": data.get("image_posts", []), - } - counts["rows_processed"] += len(manifest["image_artist_assignments"]) - counts["rows_processed"] += len(manifest["image_tag_associations"]) - counts["rows_processed"] += len(manifest["series_pages"]) - counts["rows_processed"] += len(manifest["image_posts"]) - if not dry_run: - manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) - - return counts diff --git a/backend/app/services/migrators/ml_queue.py b/backend/app/services/migrators/ml_queue.py deleted file mode 100644 index 6d9e92f..0000000 --- a/backend/app/services/migrators/ml_queue.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Queue every migrated image_record with no embedding for ML re-processing.""" -from __future__ import annotations - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import ImageRecord - - -async def queue_all_unprocessed_async(db: AsyncSession) -> int: - """Find every ImageRecord with siglip_embedding IS NULL, fire - tag_and_embed.delay(id) for each. Returns count queued. - """ - from ...tasks.ml import tag_and_embed - - rows = (await db.execute( - select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None)) - )).scalars().all() - for image_id in rows: - tag_and_embed.delay(image_id) - return len(rows) diff --git a/backend/app/services/migrators/tag_apply.py b/backend/app/services/migrators/tag_apply.py deleted file mode 100644 index 3f15c88..0000000 --- a/backend/app/services/migrators/tag_apply.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Apply the IR tag manifest after FC's filesystem scan. - -Reads /images/_migration_state/ir_tag_manifest.json and joins each entry -to an ImageRecord row by sha256 (which exists after the operator runs -FC's filesystem scan over the mounted IR images dir). - -- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug). -- image_tag_associations → image_tag insert (idempotent). -- series_pages → series_page insert (idempotent on image_id unique). -- image_posts (schema v2) → Source + Post + ImageProvenance restore. - -Unmatched sha256s are logged into the result's `unmatched` list so the -Celery task can drop them into MigrationRun.metadata for the operator -to inspect. -""" -from __future__ import annotations - -import json -from datetime import datetime -from pathlib import Path - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import ( - Artist, - ImageProvenance, - ImageRecord, - Post, - SeriesPage, - Source, - Tag, - TagKind, - image_tag, -) -from ...utils.slug import slugify -from .ir_ingest import manifest_path - -# Per-platform artist-profile URL — used as Source.url when restoring -# IR PostMetadata into FC. Must cover every platform that -# backend/app/services/extension_service.py:_PLATFORM_PATTERNS -# recognizes; an entry missing here silently drops ALL PostMetadata for -# that platform during phase 4 (operator hit this 2026-05-25: -# DeviantArt + Pixiv posts in the IR migration produced empty -# ImageProvenance because they fell through this table). -# -# Pixiv caveat: the real profile URL takes a numeric user_id -# (https://www.pixiv.net/users/12345), but IR's PostMetadata.artist -# stores the display name not the id. We use the slugified name here -# so we preserve the artist→post→image linkage; the resulting Source.url -# won't resolve in a browser and the operator may want to manually fix -# it via Settings → Subscriptions once the migration lands. -_PLATFORM_PROFILE_URL = { - "patreon": "https://www.patreon.com/{slug}", - "subscribestar": "https://www.subscribestar.com/{slug}", - "hentaifoundry": "https://www.hentai-foundry.com/user/{slug}", - "deviantart": "https://www.deviantart.com/{slug}", - "pixiv": "https://www.pixiv.net/users/{slug}", -} - - -def _profile_url(platform: str, artist_slug: str) -> str | None: - fmt = _PLATFORM_PROFILE_URL.get(platform) - return fmt.format(slug=artist_slug) if fmt else None - - -async def _find_or_create_source( - db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool, -) -> int | None: - existing = (await db.execute( - select(Source.id).where( - Source.artist_id == artist_id, - Source.platform == platform, - Source.url == url, - ) - )).scalar_one_or_none() - if existing is not None: - return existing - if dry_run: - return None - s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False) - db.add(s) - await db.flush() - return s.id - - -async def _find_or_create_post( - db: AsyncSession, *, - source_id: int, external_post_id: str, - title: str | None, description: str | None, post_url: str | None, - post_date_iso: str | None, attachment_count: int, dry_run: bool, -) -> int | None: - existing = (await db.execute( - select(Post.id).where( - Post.source_id == source_id, - Post.external_post_id == external_post_id, - ) - )).scalar_one_or_none() - if existing is not None: - return existing - if dry_run: - return None - post_date = None - if post_date_iso: - post_date = datetime.fromisoformat(post_date_iso) - p = Post( - source_id=source_id, - external_post_id=external_post_id, - post_title=title, - description=description, - post_url=post_url, - post_date=post_date, - attachment_count=attachment_count, - raw_metadata={"migrated_from": "imagerepo"}, - ) - db.add(p) - await db.flush() - return p.id - - -async def _ensure_provenance( - db: AsyncSession, *, - image_id: int, post_id: int, source_id: int, dry_run: bool, -) -> bool: - """Returns True if a new ImageProvenance row was inserted. - - Also sets ImageRecord.primary_post_id to this post if the image - doesn't already have one — preserves any primary_post_id already - assigned at download time by the importer (don't clobber). This is - the linkage gallery_service.py uses to surface Post.post_date as - the image's effective date for sort/group/jump/neighbor nav. - """ - existing = (await db.execute( - select(ImageProvenance.id).where( - ImageProvenance.image_record_id == image_id, - ImageProvenance.post_id == post_id, - ImageProvenance.source_id == source_id, - ) - )).scalar_one_or_none() - - # Whether-or-not the provenance row already exists, ensure the - # image's primary_post_id is set so the gallery date-coalesce works. - # Idempotent: only writes when currently NULL. - if not dry_run: - await db.execute( - ImageRecord.__table__.update() - .where(ImageRecord.id == image_id) - .where(ImageRecord.primary_post_id.is_(None)) - .values(primary_post_id=post_id) - ) - - if existing is not None: - return False - if dry_run: - return True - db.add(ImageProvenance( - image_record_id=image_id, post_id=post_id, source_id=source_id, - )) - await db.flush() - return True - - -def _zero_counts() -> dict: - return { - "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, - "files_copied": 0, "bytes_copied": 0, "conflicts": 0, - } - - -async def _ensure_artist_id( - db: AsyncSession, artist_name: str, dry_run: bool, -) -> int | None: - if not artist_name or not artist_name.strip(): - return None - slug = slugify(artist_name) - existing = (await db.execute( - select(Artist).where(Artist.slug == slug) - )).scalar_one_or_none() - if existing is not None: - return existing.id - if dry_run: - return None - a = Artist(name=artist_name, slug=slug, is_subscription=False) - db.add(a) - await db.flush() - return a.id - - -async def _resolve_tag_id( - db: AsyncSession, tag_name: str, tag_kind: str, -) -> int | None: - try: - kind = TagKind(tag_kind) - except ValueError: - kind = TagKind.general - row = (await db.execute( - select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind) - )).scalar_one_or_none() - return row - - -async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None: - return (await db.execute( - select(ImageRecord.id).where(ImageRecord.sha256 == sha) - )).scalar_one_or_none() - - -async def apply_async( - db: AsyncSession, - *, - images_root: Path | None = None, - dry_run: bool = False, -) -> dict: - """Apply the manifest. Returns counts + an `unmatched` list of sha256s.""" - mf_path = manifest_path(images_root) - if not mf_path.exists(): - raise FileNotFoundError(f"no IR tag manifest at {mf_path}") - - manifest = json.loads(mf_path.read_text()) - counts = _zero_counts() - unmatched: list[dict] = [] - - # 1. Artist assignments. - for entry in manifest.get("image_artist_assignments", []): - counts["rows_processed"] += 1 - img_id = await _sha_to_image_id(db, entry["sha256"]) - if img_id is None: - unmatched.append({"kind": "artist", **entry}) - counts["rows_skipped"] += 1 - continue - aid = await _ensure_artist_id(db, entry["artist_name"], dry_run) - if aid is None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - img = await db.get(ImageRecord, img_id) - if img is not None and img.artist_id != aid: - img.artist_id = aid - counts["rows_inserted"] += 1 - else: - counts["rows_skipped"] += 1 - - # 2. Tag associations. - for entry in manifest.get("image_tag_associations", []): - counts["rows_processed"] += 1 - img_id = await _sha_to_image_id(db, entry["sha256"]) - if img_id is None: - unmatched.append({"kind": "tag", **entry}) - counts["rows_skipped"] += 1 - continue - tag_id = await _resolve_tag_id( - db, entry["tag_name"], entry.get("tag_kind") or "general", - ) - if tag_id is None: - counts["rows_skipped"] += 1 - continue - # Skip if association already exists. - already = (await db.execute( - select(image_tag.c.image_record_id).where( - image_tag.c.image_record_id == img_id, - image_tag.c.tag_id == tag_id, - ) - )).first() - if already is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - await db.execute(image_tag.insert().values( - image_record_id=img_id, tag_id=tag_id, source="manual", - )) - counts["rows_inserted"] += 1 - - # 3. Series pages. - for entry in manifest.get("series_pages", []): - counts["rows_processed"] += 1 - img_id = await _sha_to_image_id(db, entry["sha256"]) - if img_id is None: - unmatched.append({"kind": "series", **entry}) - counts["rows_skipped"] += 1 - continue - series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series") - if series_tag_id is None: - counts["rows_skipped"] += 1 - continue - existing = (await db.execute( - select(SeriesPage).where(SeriesPage.image_id == img_id) - )).scalar_one_or_none() - if existing is not None: - counts["rows_skipped"] += 1 - continue - if dry_run: - counts["rows_inserted"] += 1 - continue - db.add(SeriesPage( - series_tag_id=series_tag_id, - image_id=img_id, - page_number=entry["page_number"], - )) - counts["rows_inserted"] += 1 - - # 4. Image posts (schema v2) → Source + Post + ImageProvenance. - # Restores IR PostMetadata as FC's downloader-track provenance, - # so the modal's ProvenancePanel surfaces title/description/ - # source URL/publish date the same way it does for live - # gallery-dl downloads. - for entry in manifest.get("image_posts", []): - counts["rows_processed"] += 1 - platform = entry.get("platform") - artist_name = entry.get("artist") - if not platform or not artist_name: - counts["rows_skipped"] += 1 - continue - - aid = await _ensure_artist_id(db, artist_name, dry_run) - if aid is None: - counts["rows_skipped"] += 1 - continue - - url = _profile_url(platform, slugify(artist_name)) - if url is None: - counts["rows_skipped"] += 1 - continue - - source_id = await _find_or_create_source( - db, artist_id=aid, platform=platform, url=url, dry_run=dry_run, - ) - if source_id is None: - counts["rows_skipped"] += 1 - continue - - post_id = await _find_or_create_post( - db, source_id=source_id, - external_post_id=entry.get("post_id") or "", - title=entry.get("title"), - description=entry.get("description"), - post_url=entry.get("source_url"), - post_date_iso=entry.get("published_at"), - attachment_count=entry.get("attachment_count") or 0, - dry_run=dry_run, - ) - if post_id is None: - counts["rows_skipped"] += 1 - continue - - for sha in entry.get("image_sha256s", []): - img_id = await _sha_to_image_id(db, sha) - if img_id is None: - unmatched.append({ - "kind": "post", "sha256": sha, - "post_id": entry.get("post_id"), - }) - continue - inserted = await _ensure_provenance( - db, image_id=img_id, post_id=post_id, - source_id=source_id, dry_run=dry_run, - ) - if inserted: - counts["rows_inserted"] += 1 - else: - counts["rows_skipped"] += 1 - - if not dry_run: - await db.commit() - return {"counts": counts, "unmatched": unmatched} diff --git a/backend/app/services/migrators/verify.py b/backend/app/services/migrators/verify.py deleted file mode 100644 index 52bf37c..0000000 --- a/backend/app/services/migrators/verify.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Post-migration verification: row counts + sha256 sampling.""" -from __future__ import annotations - -import hashlib -from pathlib import Path - -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from ...models import Artist, Credential, ImageRecord, Source, Tag - - -async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict: - """Return per-check status dicts. `expected` is optional row-count - assertions; checks default to status='ok' when no expected provided.""" - expected = expected or {} - results: dict[str, dict] = {} - - checks = { - "artist_subscriptions": ( - select(func.count(Artist.id)).where(Artist.is_subscription.is_(True)) - ), - "source_count": select(func.count(Source.id)), - "credential_count": select(func.count(Credential.id)), - "tag_count": select(func.count(Tag.id)), - "image_record_imported_or_downloaded": ( - select(func.count(ImageRecord.id)) - .where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"])) - ), - } - for name, stmt in checks.items(): - actual = (await db.execute(stmt)).scalar_one() - exp = expected.get(name) - status = "ok" if exp is None or exp == actual else "mismatch" - results[name] = {"status": status, "actual": int(actual), "expected": exp} - return results - - -async def verify_sha256_sample( - db: AsyncSession, *, sample_size: int = 20, -) -> dict: - """Sample N image_records; verify file exists + sha256 matches.""" - rows = (await db.execute( - select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256) - .order_by(func.random()).limit(sample_size) - )).all() - - matched = 0 - mismatched = 0 - missing = 0 - samples: list[dict] = [] - for img_id, path, expected_sha in rows: - p = Path(path) - # Sync stdlib filesystem ops are intentional: this verify pass runs - # inside a Celery task under asyncio.run; no other awaitables compete - # for the loop. Same pattern as download_service.py. - if not p.exists(): # noqa: ASYNC240 - missing += 1 - samples.append({"id": img_id, "path": path, "result": "missing"}) - continue - h = hashlib.sha256() - with p.open("rb") as f: # noqa: ASYNC230 - for chunk in iter(lambda: f.read(65536), b""): - h.update(chunk) - if h.hexdigest() == expected_sha: - matched += 1 - samples.append({"id": img_id, "result": "ok"}) - else: - mismatched += 1 - samples.append({ - "id": img_id, "path": path, "result": "mismatch", - "expected_sha": expected_sha, "actual_sha": h.hexdigest(), - }) - return { - "sample_size": len(rows), - "matched": matched, - "mismatched": mismatched, - "missing": missing, - "samples": samples, - } diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index 974427e..319d645 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -56,9 +56,17 @@ class PostFeedService: artist_id: int | None = None, platform: str | None = None, limit: int = 24, + direction: str = "older", ) -> dict: + """Paginate the feed from `cursor`. direction='older' walks back in + time (default, infinite-scroll down); direction='newer' walks forward + (scroll up in an anchored view). Items are always returned in feed + (descending) order; `next_cursor` points to the far edge in the + requested direction (null when exhausted).""" if limit < 1 or limit > 100: raise ValueError("limit must be between 1 and 100") + if direction not in ("older", "newer"): + raise ValueError("direction must be 'older' or 'newer'") sort_key = _sort_key() stmt = ( @@ -72,22 +80,37 @@ class PostFeedService: stmt = stmt.where(Source.platform == platform) if cursor: cur_ts, cur_id = decode_cursor(cursor) - stmt = stmt.where( - or_( + if direction == "older": + stmt = stmt.where(or_( sort_key < cur_ts, and_(sort_key == cur_ts, Post.id < cur_id), - ) - ) + )) + else: + stmt = stmt.where(or_( + sort_key > cur_ts, + and_(sort_key == cur_ts, Post.id > cur_id), + )) - stmt = stmt.order_by(sort_key.desc(), Post.id.desc()).limit(limit + 1) + if direction == "older": + stmt = stmt.order_by(sort_key.desc(), Post.id.desc()) + else: + stmt = stmt.order_by(sort_key.asc(), Post.id.asc()) + stmt = stmt.limit(limit + 1) rows = (await self.session.execute(stmt)).all() + has_more = len(rows) > limit + rows = rows[:limit] + if direction == "newer": + # Fetched ascending (closest-newer first); flip to feed order. + rows = list(reversed(rows)) + next_cursor: str | None = None - if len(rows) > limit: - last_post, _, _ = rows[limit - 1] - last_key = last_post.post_date or last_post.downloaded_at - next_cursor = encode_cursor(last_key, last_post.id) - rows = rows[:limit] + if has_more and rows: + # Far edge in the travel direction: oldest row going older, + # newest row going newer (rows is descending for display). + edge_post = rows[-1][0] if direction == "older" else rows[0][0] + edge_key = edge_post.post_date or edge_post.downloaded_at + next_cursor = encode_cursor(edge_key, edge_post.id) post_ids = [p.id for p, _, _ in rows] thumbs_map = await self._thumbnails_for(post_ids) @@ -99,6 +122,49 @@ class PostFeedService: ] return {"items": items, "next_cursor": next_cursor} + async def around( + self, + *, + post_id: int, + artist_id: int | None = None, + platform: str | None = None, + limit: int = 12, + ) -> dict | None: + """A window centered on `post_id`: up to `limit` newer posts + the + post + up to `limit` older posts, in feed (descending) order, with a + cursor for each end. Returns None if the post doesn't exist.""" + anchor = (await self.session.execute( + select(Post, Artist, Source) + .join(Source, Post.source_id == Source.id) + .join(Artist, Source.artist_id == Artist.id) + .where(Post.id == post_id) + )).one_or_none() + if anchor is None: + return None + anchor_post, anchor_artist, anchor_source = anchor + anchor_key = anchor_post.post_date or anchor_post.downloaded_at + anchor_cursor = encode_cursor(anchor_key, anchor_post.id) + + older = await self.scroll( + cursor=anchor_cursor, artist_id=artist_id, platform=platform, + limit=limit, direction="older", + ) + newer = await self.scroll( + cursor=anchor_cursor, artist_id=artist_id, platform=platform, + limit=limit, direction="newer", + ) + thumbs_map = await self._thumbnails_for([anchor_post.id]) + atts_map = await self._attachments_for([anchor_post.id]) + anchor_item = self._to_dict( + anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map, + ) + return { + "items": newer["items"] + [anchor_item] + older["items"], + "cursor_older": older["next_cursor"], + "cursor_newer": newer["next_cursor"], + "anchor_id": anchor_post.id, + } + async def get_post(self, post_id: int) -> dict | None: row = (await self.session.execute( select(Post, Artist, Source) diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 8bd321b..1c4cf22 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -10,7 +10,14 @@ from PIL import Image from sqlalchemy import and_, delete, or_, select, update from ..celery_app import celery -from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun +from ..models import ( + DownloadEvent, + ImageRecord, + ImportSettings, + ImportTask, + Source, + TaskRun, +) from ..utils.phash import compute_phash from ._sync_engine import sync_session_factory as _sync_session_factory @@ -34,6 +41,13 @@ ARCHIVE_STUCK_THRESHOLD_MINUTES = 40 # flip to terminal 'failed' and never enter this loop. MAX_RECOVERY_ATTEMPTS = 3 ORPHAN_PENDING_THRESHOLD_MINUTES = 30 + +# DownloadEvent (pending|running) recovery threshold. download_source has +# time_limit=1200s (20 min); 30 min is 10 min past that, so a legitimately- +# running task is never killed by the sweep. Operator-confirmed 2026-05-29 +# after 43 sources stranded at "last check never" by the in-flight guard. +DOWNLOAD_STALL_THRESHOLD_MINUTES = 30 + OLD_TASK_DAYS = 7 PHASH_PAGE = 500 VERIFY_PAGE = 200 @@ -448,6 +462,65 @@ def verify_integrity() -> int: return total +@celery.task(name="backend.app.tasks.maintenance.recover_stalled_download_events") +def recover_stalled_download_events() -> int: + """Recover DownloadEvent rows stuck pending/running past the worker hard kill. + + The scan tick (scheduler_service.select_due_sources → + tasks.scan._tick_due_sources_async) inserts DownloadEvent(status='pending') + and fires download_source.delay(). If that task dies before finalizing the + event — worker OOM/SIGKILL, lost task, or a gallery-dl that didn't unwind + on the 1200s hard time_limit — the event stays in-flight forever. The next + tick then skips that source because of the in-flight guard (scan.py:168) + and Source.last_checked_at never updates; the operator sees "last check + never" in the Subscriptions health column, permanently. + + This sweep flips matching events to 'error', stamps each affected Source's + last_checked_at + last_error and bumps consecutive_failures (once per + source, not per event — backoff is exponential on that count so an N-event + bump would inflate the next interval by 2^N for no reason). The source + becomes re-queueable on the next tick and the health dot goes amber. + + Operator-confirmed 2026-05-29 (43-row strand pile in production). + """ + SessionLocal = _sync_session_factory() + now = datetime.now(UTC) + cutoff = now - timedelta(minutes=DOWNLOAD_STALL_THRESHOLD_MINUTES) + msg = "stranded by recovery sweep (no terminal status after time_limit)" + with SessionLocal() as session: + # UPDATE...RETURNING the source_ids in one round trip — keeps us off + # the psycopg 65535-param ceiling that SELECT-then-UPDATE-WHERE-IN + # would hit on a large strand pile. + result = session.execute( + update(DownloadEvent) + .where(DownloadEvent.status.in_(["pending", "running"])) + .where(DownloadEvent.started_at < cutoff) + .values(status="error", finished_at=now, error=msg) + .returning(DownloadEvent.source_id) + ) + returned = result.all() + if not returned: + session.commit() + return 0 + events_recovered = len(returned) + source_ids = list({row.source_id for row in returned}) + session.execute( + update(Source) + .where(Source.id.in_(source_ids)) + .values( + consecutive_failures=Source.consecutive_failures + 1, + last_error=msg, + last_checked_at=now, + ) + ) + session.commit() + log.info( + "recover_stalled_download_events: recovered %d events across %d sources", + events_recovered, len(source_ids), + ) + return events_recovered + + @celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events") def cleanup_old_download_events() -> int: """FC-3d: delete terminal DownloadEvent rows older than the configured diff --git a/backend/app/tasks/migration.py b/backend/app/tasks/migration.py deleted file mode 100644 index aec5102..0000000 --- a/backend/app/tasks/migration.py +++ /dev/null @@ -1,169 +0,0 @@ -"""FC-5 run_migration Celery task. - -Dispatches to the right migrator based on `kind`. Updates MigrationRun -row's status/counts/finished_at as it runs. Failures set status='error' -with the error message preserved. - -kinds: gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup -(backup + rollback retired 2026-05-24 → see /api/system/backup/*) -""" -from __future__ import annotations - -import asyncio -import logging -from datetime import UTC, datetime -from pathlib import Path - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from ..celery_app import celery -from ..models import MigrationRun -from ..services.credential_crypto import CredentialCrypto -from ..services.migrators import cleanup as cleanup_mod -from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify -from ._async_session import async_session_factory - -log = logging.getLogger(__name__) - -IMAGES_ROOT = Path("/images") -_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64" - - -async def _update_run( - db: AsyncSession, run_id: int, *, - status: str | None = None, counts: dict | None = None, - error: str | None = None, finished_at: datetime | None = None, - metadata_patch: dict | None = None, -) -> None: - run = (await db.execute( - select(MigrationRun).where(MigrationRun.id == run_id) - )).scalar_one() - if status is not None: - run.status = status - if counts is not None: - run.counts = counts - if error is not None: - run.error = error - if finished_at is not None: - run.finished_at = finished_at - if metadata_patch: - run.metadata_ = {**(run.metadata_ or {}), **metadata_patch} - await db.commit() - - -async def _run_async(run_id: int, kind: str, params: dict) -> dict: - factory, engine = async_session_factory() - try: - async with factory() as db: - await _update_run(db, run_id, status="running") - try: - if kind in ("backup", "rollback"): - raise ValueError( - f"kind {kind!r} retired in FC-3h; " - "use /api/system/backup/* instead" - ) - - elif kind == "gs_ingest": - fc_crypto = CredentialCrypto(_KEY_PATH) - counts = await gs_ingest.migrate_async( - db, data=params["data"], - fc_crypto=fc_crypto, - dry_run=params.get("dry_run", False), - ) - await _update_run( - db, run_id, status="ok", counts=counts, - finished_at=datetime.now(UTC), - ) - return counts - - elif kind == "ir_ingest": - counts = await ir_ingest.migrate_async( - db, data=params["data"], - images_root=IMAGES_ROOT, - dry_run=params.get("dry_run", False), - ) - await _update_run( - db, run_id, status="ok", counts=counts, - finished_at=datetime.now(UTC), - ) - return counts - - elif kind == "tag_apply": - result = await tag_apply.apply_async( - db, images_root=IMAGES_ROOT, - dry_run=params.get("dry_run", False), - ) - await _update_run( - db, run_id, status="ok", - counts=result["counts"], - finished_at=datetime.now(UTC), - metadata_patch={"unmatched": result["unmatched"]}, - ) - return result - - elif kind == "ml_queue": - count = await ml_queue.queue_all_unprocessed_async(db) - await _update_run( - db, run_id, status="ok", - counts={"rows_processed": count, "rows_inserted": 0, - "rows_skipped": 0, "files_copied": 0, - "bytes_copied": 0, "conflicts": 0}, - finished_at=datetime.now(UTC), - ) - return {"queued": count} - - elif kind == "verify": - checks = await verify.verify_async(db, expected=params.get("expected")) - sample = await verify.verify_sha256_sample( - db, sample_size=params.get("sample_size", 20), - ) - await _update_run( - db, run_id, status="ok", - counts={"rows_processed": sample["sample_size"], - "rows_inserted": 0, "rows_skipped": 0, - "files_copied": 0, "bytes_copied": 0, - "conflicts": sample["mismatched"] + sample["missing"]}, - finished_at=datetime.now(UTC), - metadata_patch={"checks": checks, "sample": sample}, - ) - return {"checks": checks, "sample": sample} - - elif kind == "cleanup": - slug = params.get("slug") - if not slug: - raise ValueError("cleanup requires params.slug") - result = await cleanup_mod.cleanup_artist_async( - db, slug=slug, images_root=IMAGES_ROOT, - dry_run=params.get("dry_run", False), - source_path_prefix=params.get("source_path_prefix"), - ) - await _update_run( - db, run_id, status="ok", - counts=result["counts"], - finished_at=datetime.now(UTC), - metadata_patch={ - "artist": result["artist"], - "summary": result["summary"], - }, - ) - return result - - else: - raise ValueError(f"unknown kind: {kind}") - - except Exception as exc: - log.exception("migration kind=%s failed", kind) - await _update_run( - db, run_id, status="error", error=str(exc), - finished_at=datetime.now(UTC), - ) - raise - finally: - await engine.dispose() - - -@celery.task(name="backend.app.tasks.migration.run_migration", bind=True, acks_late=True) -def run_migration(self, run_id: int, kind: str, params: dict) -> dict: - """FC-5: dispatch a migration kind. Updates MigrationRun row as it goes.""" - return asyncio.run(_run_async(run_id, kind, params)) diff --git a/frontend/package.json b/frontend/package.json index d68122c..5e230c5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,6 +26,8 @@ "vue-tsc": "^2.0.0", "vite-plugin-vuetify": "^2.0.0", "sass": "^1.71.0", - "vitest": "^2.1.0" + "vitest": "^2.1.0", + "@vue/test-utils": "^2.4.0", + "happy-dom": "^15.0.0" } } diff --git a/frontend/src/components/PipelineStatusChip.vue b/frontend/src/components/PipelineStatusChip.vue new file mode 100644 index 0000000..0b026c0 --- /dev/null +++ b/frontend/src/components/PipelineStatusChip.vue @@ -0,0 +1,146 @@ + + + + + diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index b2d7939..07b5983 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -8,6 +8,7 @@ {{ health.icon }} +