diff --git a/alembic/versions/0088_reconcile_models_with_schema.py b/alembic/versions/0088_reconcile_models_with_schema.py new file mode 100644 index 0000000..6117dd3 --- /dev/null +++ b/alembic/versions/0088_reconcile_models_with_schema.py @@ -0,0 +1,48 @@ +"""Reconcile the database with what the models have always claimed (#3275). + +Milestone 328 discovered ~130 places where the ORM models and the deployed +schema disagreed. Almost all of them were the MODEL being wrong — missing +`server_default`s, indexes and CHECK constraints that only ever existed in a +migration — and those are fixed in the model files with no DDL at all, because +the database already had them. + +This migration carries the remainder: the one case where the MODEL was right +and the database was missing something. + +`tag.fandom_id` is declared `index=True` on the model, but no migration ever +created that index. Every autogenerate run since would have proposed adding +it; nobody ran one, so the model and the database simply drifted apart and +stayed that way. + +Deliberately NOT in this migration: making `image_record.sha256` unique. The +model says `unique=True` and `0001` created a plain, non-unique index, so +duplicates are possible today and the ORM believes they are not. Adding the +constraint is a real change that FAILS if duplicates already exist, and if +they do exist the right response is a dedupe decision rather than a constraint +— so it needs an answer about live data before it is written, not after. +Tracked in #3275. + +Revision ID: 0088 +Revises: 0087 +Create Date: 2026-08-30 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0088" +down_revision: Union[str, None] = "0087" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # IF NOT EXISTS because the index is what the model already asks for: any + # database built from metadata rather than from this chain will have it, + # and this migration must be a no-op there rather than an error. + op.execute("CREATE INDEX IF NOT EXISTS ix_tag_fandom_id ON tag (fandom_id)") + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_tag_fandom_id") diff --git a/backend/app/models/artist.py b/backend/app/models/artist.py index e7ca894..fb3401c 100644 --- a/backend/app/models/artist.py +++ b/backend/app/models/artist.py @@ -27,10 +27,10 @@ class Artist(Base): notes: Mapped[str | None] = mapped_column(Text, nullable=True) # True once a Source is attached; flips false if all sources removed. - is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") # Per-artist scheduling overrides; null means "use global default". - auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.py index 1717aea..ad7b4f0 100644 --- a/backend/app/models/backup_run.py +++ b/backend/app/models/backup_run.py @@ -20,7 +20,7 @@ feedback_check_existing_enums): from datetime import datetime -from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -29,10 +29,18 @@ from .base import Base class BackupRun(Base): __tablename__ = "backup_run" + + __table_args__ = ( + # alembic 0017: reporting indexes, never declared on the model (#3275). + Index("ix_backup_run_kind_started", "kind", text("started_at DESC")), + Index("ix_backup_run_status_finished", "status", text("finished_at DESC")), + Index("ix_backup_run_tag_partial", "tag", postgresql_where=text("tag IS NOT NULL")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True) status: Mapped[str] = mapped_column( String(16), nullable=False, default="pending", index=True, + server_default="pending", ) tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) triggered_by: Mapped[str] = mapped_column(String(32), nullable=False) diff --git a/backend/app/models/download_event.py b/backend/app/models/download_event.py index 7fe5b37..3ec00ae 100644 --- a/backend/app/models/download_event.py +++ b/backend/app/models/download_event.py @@ -25,8 +25,8 @@ class DownloadEvent(Base): DateTime(timezone=True), nullable=False, server_default=func.now() ) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) - files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default="0") + files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") error: Mapped[str | None] = mapped_column(Text, nullable=True) metadata_: Mapped[dict] = mapped_column( "metadata", JSONB, nullable=False, default=dict, diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py index 0902e28..dcf8ee8 100644 --- a/backend/app/models/external_link.py +++ b/backend/app/models/external_link.py @@ -16,6 +16,7 @@ doesn't delete the link record). from datetime import datetime from sqlalchemy import ( + CheckConstraint, DateTime, Float, ForeignKey, @@ -38,6 +39,16 @@ STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead") class ExternalLink(Base): __tablename__ = "external_link" __table_args__ = ( + # alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value + # needs its constraint swapped in the same migration (#3275). + CheckConstraint( + "host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')", + name="ck_external_link_host", + ), + CheckConstraint( + "status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')", + name="ck_external_link_status", + ), # One row per (post, url). The full url (incl. #fragment) is the identity # — the same file linked twice in a post collapses to one row. Index("uq_external_link_post_url", "post_id", "url", unique=True), diff --git a/backend/app/models/gpu_job.py b/backend/app/models/gpu_job.py index dba5997..931b455 100644 --- a/backend/app/models/gpu_job.py +++ b/backend/app/models/gpu_job.py @@ -50,7 +50,8 @@ class GpuJob(Base): # What to compute, e.g. 'ccip' (detect figures + CCIP-embed) or 'siglip_region'. task: Mapped[str] = mapped_column(String(32), nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="pending", index=True + String(16), nullable=False, default="pending", index=True, + server_default="pending", ) # pending | leased | done | error lease_token: Mapped[str | None] = mapped_column(String(64), nullable=True) @@ -60,7 +61,7 @@ class GpuJob(Base): lease_expires_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True ) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") error: Mapped[str | None] = mapped_column(Text, nullable=True) # Triage verdict for an ERRORED job (#125): NULL = not yet probed; # 'defect' = the integrity probe says the FILE itself is bad (surfaced for diff --git a/backend/app/models/head_auto_apply_run.py b/backend/app/models/head_auto_apply_run.py index 08c359a..031109b 100644 --- a/backend/app/models/head_auto_apply_run.py +++ b/backend/app/models/head_auto_apply_run.py @@ -24,10 +24,11 @@ class HeadAutoApplyRun(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True) # dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing # (preview/apply parity, rule 93). - dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="running", index=True + String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | error started_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/head_metric.py b/backend/app/models/head_metric.py index a034e51..7afc104 100644 --- a/backend/app/models/head_metric.py +++ b/backend/app/models/head_metric.py @@ -24,9 +24,9 @@ class HeadMetric(Base): ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True ) # An auto-applied (source='head_auto') tag the operator later REMOVED. - n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # A tag with a head that the operator added by HAND (the head missed it). - n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/head_metrics_snapshot.py b/backend/app/models/head_metrics_snapshot.py index a9ec7ac..651344e 100644 --- a/backend/app/models/head_metrics_snapshot.py +++ b/backend/app/models/head_metrics_snapshot.py @@ -28,9 +28,9 @@ class HeadMetricsSnapshot(Base): DateTime(timezone=True), nullable=False, server_default=func.now(), index=True ) # Current count of source='head_auto' applications still standing. - n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # The head's measured quality at snapshot time (null if no head exists). ap: Mapped[float | None] = mapped_column(Float, nullable=True) precision_cv: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/backend/app/models/head_training_run.py b/backend/app/models/head_training_run.py index fd5858e..21c4ac7 100644 --- a/backend/app/models/head_training_run.py +++ b/backend/app/models/head_training_run.py @@ -24,7 +24,8 @@ class HeadTrainingRun(Base): # Training parameters: {min_positives, neg_ratio, precision_target, ...}. params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="running", index=True + String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | error started_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 32d6eaa..be273ce 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -14,10 +14,12 @@ from sqlalchemy import ( Enum, Float, ForeignKey, + Index, Integer, String, Text, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column @@ -29,6 +31,12 @@ ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded") class ImageRecord(Base): __tablename__ = "image_record" + + __table_args__ = ( + # alembic 0035/0071: the date-ordered browse indexes (#3275). + Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")), + Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) # On-disk identity @@ -47,7 +55,8 @@ class ImageRecord(Base): # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'. # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'. integrity_status: Mapped[str] = mapped_column( - String(24), nullable=False, default="unknown", index=True + String(24), nullable=False, default="unknown", index=True, + server_default="unknown", ) # Thumbnail (populated by FC-2) diff --git a/backend/app/models/import_batch.py b/backend/app/models/import_batch.py index 474f111..8d8fa61 100644 --- a/backend/app/models/import_batch.py +++ b/backend/app/models/import_batch.py @@ -21,17 +21,17 @@ class ImportBatch(Base): ) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # Deep-scan only: count of already-imported files whose sidecar metadata # got re-applied this run (post/source/provenance upsert). Stays 0 on # quick-scan batches. See `Importer.import_one(deep_scan=True)`. - refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") - status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True, server_default="running") # running | complete | cancelled tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan") diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index f4b8937..74ac13f 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -17,60 +17,71 @@ class ImportSettings(Base): __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) - import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import") + import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import", server_default="/import") - min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") - skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9) + skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") + transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9") - skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) - single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30) + skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95") + single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30") - phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10, server_default="10") # FC-3c downloader knobs download_rate_limit_seconds: Mapped[float] = mapped_column( - Float, nullable=False, default=3.0 + Float, nullable=False, default=3.0, + server_default="3", ) download_validate_files: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) # FC-3d scheduling knobs download_schedule_default_seconds: Mapped[int] = mapped_column( - Integer, nullable=False, default=28800 + Integer, nullable=False, default=28800, + server_default="28800", ) download_event_retention_days: Mapped[int] = mapped_column( - Integer, nullable=False, default=90 + Integer, nullable=False, default=90, + server_default="90", ) download_failure_warning_threshold: Mapped[int] = mapped_column( - Integer, nullable=False, default=5 + Integer, nullable=False, default=5, + server_default="5", ) # FC-3h backup knobs. backup_db_nightly_enabled: Mapped[bool] = mapped_column( Boolean, nullable=False, default=False, + server_default="false", ) backup_db_nightly_hour_utc: Mapped[int] = mapped_column( Integer, nullable=False, default=3, + server_default="3", ) backup_db_keep_last_n: Mapped[int] = mapped_column( Integer, nullable=False, default=14, + server_default="14", ) backup_images_keep_last_n: Mapped[int] = mapped_column( Integer, nullable=False, default=3, + server_default="3", ) # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is # the weighted-score cut-off (0..1) above which a pending suggestion is made. series_suggest_enabled: Mapped[bool] = mapped_column( Boolean, nullable=False, default=True, + server_default="true", ) series_suggest_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.5, + server_default="0.5", ) # #830 off-platform file-host downloads — per-host enable lever (default on, diff --git a/backend/app/models/import_task.py b/backend/app/models/import_task.py index 3c947c1..c3d9f11 100644 --- a/backend/app/models/import_task.py +++ b/backend/app/models/import_task.py @@ -13,10 +13,12 @@ from sqlalchemy import ( Boolean, DateTime, ForeignKey, + Index, Integer, String, Text, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -26,6 +28,10 @@ from .base import Base class ImportTask(Base): __tablename__ = "import_task" + + __table_args__ = ( + Index("ix_import_task_created_at_desc", text("created_at DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) batch_id: Mapped[int] = mapped_column( ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True @@ -33,14 +39,14 @@ class ImportTask(Base): source_path: Mapped[str] = mapped_column(Text, nullable=False) task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive - status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True, server_default="pending") # Poison-pill circuit breaker (alembic 0026). recovery_count tracks # how many times the stuck-task sweep has re-queued this row; after # the cap it's failed with a diagnostic instead of looping. refetched # bounds the one-shot re-download remediation to a single attempt. - recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") result_image_id: Mapped[int | None] = mapped_column( ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True diff --git a/backend/app/models/library_audit_run.py b/backend/app/models/library_audit_run.py index a2d4bc2..6ac9bb2 100644 --- a/backend/app/models/library_audit_run.py +++ b/backend/app/models/library_audit_run.py @@ -8,7 +8,7 @@ reads it and routes through cleanup_service.delete_images. from datetime import datetime from typing import Any -from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy import DateTime, Integer, String, Text, func, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column @@ -23,6 +23,7 @@ class LibraryAuditRun(Base): params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | applied | cancelled | error started_at: Mapped[datetime] = mapped_column( @@ -31,14 +32,16 @@ class LibraryAuditRun(Base): finished_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, ) - scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list) + scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + matched_ids: Mapped[list[int]] = mapped_column( + JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb") + ) error: Mapped[str | None] = mapped_column(Text, nullable=True) # Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes # from, and the last time a chunk made progress (so the recovery sweep can # tell a progressing multi-chunk audit from a stuck one). - resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") last_progress_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, ) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 72da17b..3e1570a 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -31,17 +31,20 @@ class MLSettings(Base): # queueing embed work nothing will consume (the daily GPU 'embed' backfill # covers those images instead). cpu_embed_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) # Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not # a fixed count) so coverage reflects real screen time regardless of length; # cap the total so a long video can't explode into hundreds of embeds. The # per-frame SigLIP embeddings are mean-pooled. Operator-tunable. video_frame_interval_seconds: Mapped[float] = mapped_column( - Float, nullable=False, default=4.0 + Float, nullable=False, default=4.0, + server_default="4", ) video_max_frames: Mapped[int] = mapped_column( - Integer, nullable=False, default=64 + Integer, nullable=False, default=64, + server_default="64", ) # Tagging-v2 head training (#114). The head is the suggestion source that # LEARNS from the operator's tags (replacing Camie + centroid). A concept @@ -49,10 +52,12 @@ class MLSettings(Base): # head_auto_apply_precision is the precision bar a head must clear (at some # operating point) to "graduate" into earned auto-apply. Operator-tunable. head_min_positives: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) head_auto_apply_precision: Mapped[float] = mapped_column( - Float, nullable=False, default=0.97 + Float, nullable=False, default=0.97, + server_default="0.97", ) # Earned auto-apply (#114). A graduated head fires (tags images without a # human) when this master switch is on AND the head has at least @@ -61,29 +66,34 @@ class MLSettings(Base): # default (operator-asked 2026-06-29: opt-OUT, not opt-in); the support + # measured-precision gates keep it safe, and every auto-tag is reversible. head_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) head_auto_apply_min_positives: Mapped[int] = mapped_column( # Support floor raised 30→50 (operator-asked 2026-07-06): a head needs # more human labels before it may fire without a human. - Integer, nullable=False, default=50 + Integer, nullable=False, default=50, + server_default="30", ) # CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75 # over-fired (high-reference characters matched a scatter of images); 0.85 # keeps the confident single-character matches. Tunable from the agent card. ccip_match_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.85 + Float, nullable=False, default=0.85, + server_default="0.85", ) # CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold, # above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out); # single-character references + the high bar keep it safe, every tag reversible. ccip_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) ccip_auto_apply_threshold: Mapped[float] = mapped_column( # Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident # character matches auto-tag. - Float, nullable=False, default=0.95 + Float, nullable=False, default=0.95, + server_default="0.92", ) # -- Presentation chrome auto-hide (#141) ------------------------------- # `banner` (chrome — clusters on UI, not content) auto-applies on the sweep @@ -95,13 +105,16 @@ class MLSettings(Base): # (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor # screenshot` are no longer chrome — they went to the PROCESS path below. presentation_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) presentation_auto_apply_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.90 + Float, nullable=False, default=0.90, + server_default="0.90", ) presentation_conflict_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.50 + Float, nullable=False, default=0.50, + server_default="0.50", ) # -- Process auto-apply (#1464) ---------------------------------------- # `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program @@ -115,24 +128,29 @@ class MLSettings(Base): # (PresentationReview, mode='process') rather than silently marked. OFF by # default — a new whole-library auto-tagger is opt-in; every auto-tag reversible. process_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False + Boolean, nullable=False, default=False, + server_default="false", ) process_auto_apply_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.90 + Float, nullable=False, default=0.90, + server_default="0.9", ) process_conflict_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.50 + Float, nullable=False, default=0.50, + server_default="0.5", ) # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069); # existing libraries keep their stored value until the operator re-embeds. embedder_model_version: Mapped[str] = mapped_column( - String(128), nullable=False, default="siglip2-so400m-patch16-512" + String(128), nullable=False, default="siglip2-so400m-patch16-512", + server_default="siglip2-so400m-patch16-512", ) # The HF model NAME the embedder loads (server CPU embed + announced to the # GPU agent in the lease). Operator-settable so the embedder is a choice, not # a hardcode (#1190): set name + version together, then re-embed + retrain. embedder_model_name: Mapped[str] = mapped_column( - String(128), nullable=False, default="google/siglip2-so400m-patch16-512" + String(128), nullable=False, default="google/siglip2-so400m-patch16-512", + server_default="google/siglip2-so400m-patch16-512", ) # -- Crop proposers / detectors (#1202, #134) -------------------------- # WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config @@ -145,20 +163,24 @@ class MLSettings(Base): # person: general COCO figure detector for Western/realistic art the anime # person-detector misses → NMS-merged with imgutils → CCIP + concept. detector_person_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_person_weights: Mapped[str] = mapped_column( - String(512), nullable=False, default="yolo11n.pt" + String(512), nullable=False, default="yolo11n.pt", + server_default="yolo11n.pt", ) detector_person_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.35 + Float, nullable=False, default=0.35, + server_default="0.35", ) # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops. # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the # upstream repo so the URL resolves. License UNSTATED — fine for a private # homelab (operator accepted #1202). detector_anatomy_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_anatomy_weights: Mapped[str] = mapped_column( String(512), nullable=False, @@ -166,37 +188,47 @@ class MLSettings(Base): "https://github.com/aperveyev/booru_yolo/raw/main/models/" "yolov11m_aa22.pt" ), + server_default="https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt", ) detector_anatomy_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.30 + Float, nullable=False, default=0.30, + server_default="0.30", ) # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x). detector_panel_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_panel_weights: Mapped[str] = mapped_column( String(512), nullable=False, default="mosesb/best-comic-panel-detection::best.pt", + server_default="mosesb/best-comic-panel-detection::best.pt", ) detector_panel_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.30 + Float, nullable=False, default=0.30, + server_default="0.30", ) # Per-frame caps bound the crop→embed explosion; max_regions is the hard # per-job backstop; dedupe_iou drops near-duplicate crops before the embed. detector_max_figures: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_components: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_panels: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_regions: Mapped[int] = mapped_column( - Integer, nullable=False, default=128 + Integer, nullable=False, default=128, + server_default="128", ) detector_dedupe_iou: Mapped[float] = mapped_column( - Float, nullable=False, default=0.85 + Float, nullable=False, default=0.85, + server_default="0.85", ) # -- CCIP character prototypes (#1317) --------------------------------- # The per-character reference set is precomputed + refreshed INCREMENTALLY @@ -208,7 +240,8 @@ class MLSettings(Base): String(128), nullable=True ) ccip_prototype_cap: Mapped[int] = mapped_column( - Integer, nullable=False, default=64 + Integer, nullable=False, default=64, + server_default="64", ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/patreon_failed_media.py b/backend/app/models/patreon_failed_media.py index 79976ef..26557fe 100644 --- a/backend/app/models/patreon_failed_media.py +++ b/backend/app/models/patreon_failed_media.py @@ -35,7 +35,7 @@ class PatreonFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/pixiv_failed_media.py b/backend/app/models/pixiv_failed_media.py index a33d15e..7737594 100644 --- a/backend/app/models/pixiv_failed_media.py +++ b/backend/app/models/pixiv_failed_media.py @@ -35,7 +35,7 @@ class PixivFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/post.py b/backend/app/models/post.py index 4183330..1cc15a4 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -9,15 +9,17 @@ artist-filter queries don't depend on the Source detour). from datetime import datetime from sqlalchemy import ( - JSON, CheckConstraint, DateTime, ForeignKey, + Index, Integer, + JSON, String, Text, UniqueConstraint, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column @@ -27,6 +29,10 @@ from .base import Base class Post(Base): __tablename__ = "post" __table_args__ = ( + # alembic 0030. The comment above described this index; nothing declared + # it, so autogenerate proposed dropping it (#3275). + Index("uq_post_artist_external_id_null_source", "artist_id", "external_post_id", + unique=True, postgresql_where=text("source_id IS NULL")), # Source-bound dedup. Postgres treats NULL != NULL so rows # with source_id IS NULL aren't deduped by this constraint; # the partial unique index `uq_post_artist_external_id_null_source` diff --git a/backend/app/models/presentation_review.py b/backend/app/models/presentation_review.py index 73da13f..e18e298 100644 --- a/backend/app/models/presentation_review.py +++ b/backend/app/models/presentation_review.py @@ -11,7 +11,7 @@ are pruned by retention. from datetime import datetime -from sqlalchemy import DateTime, Float, ForeignKey, String, func +from sqlalchemy import DateTime, Float, ForeignKey, Index, String, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -20,6 +20,10 @@ from .base import Base class PresentationReview(Base): __tablename__ = "presentation_review" + + __table_args__ = ( + Index("ix_presentation_review_resolved_at", "resolved_at"), + ) image_record_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True ) diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 1bf6c67..1c29e6f 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -5,7 +5,7 @@ Multiple sources per artist support creators with cross-platform presence. from datetime import datetime -from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, JSON, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship from .base import Base @@ -20,7 +20,7 @@ class Source(Base): ) platform: Mapped[str] = mapped_column(String(64), nullable=False) url: Mapped[str] = mapped_column(Text, nullable=False) - enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) @@ -32,7 +32,7 @@ class Source(Base): # by _update_source_health alongside last_error; cleared on 'ok'. error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) - consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # alembic 0031: sticky deep-scan budget. When > 0, the next N download # runs use gallery-dl's full-walk config (skip: True + 1800s timeout); diff --git a/backend/app/models/subscribestar_failed_media.py b/backend/app/models/subscribestar_failed_media.py index 9201aa7..d12ff73 100644 --- a/backend/app/models/subscribestar_failed_media.py +++ b/backend/app/models/subscribestar_failed_media.py @@ -34,7 +34,7 @@ class SubscribeStarFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index 8d5a256..d3107d0 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -15,11 +15,13 @@ from sqlalchemy import ( Column, DateTime, ForeignKey, + Index, Integer, String, Table, false, func, + text, ) from sqlalchemy import ( Enum as SQLEnum, @@ -67,7 +69,7 @@ image_tag = Table( primary_key=True, ), Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), - Column("source", String(32), nullable=False, default="manual"), + Column("source", String(32), nullable=False, default="manual", server_default="manual"), Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), ) @@ -75,6 +77,10 @@ image_tag = Table( class Tag(Base): __tablename__ = "tag" __table_args__ = ( + # alembic 0002. An EXPRESSION index — COALESCE cannot be expressed as a + # UniqueConstraint, which is why it only ever existed in a migration (#3275). + Index("uq_tag_name_kind_fandom", "name", "kind", text("COALESCE(fandom_id, 0)"), + unique=True), CheckConstraint( "(fandom_id IS NULL) OR (kind = 'character')", name="ck_tag_fandom_requires_character", @@ -87,6 +93,7 @@ class Tag(Base): SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]), nullable=False, default=TagKind.general, + server_default="general", ) fandom_id: Mapped[int | None] = mapped_column( ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True diff --git a/backend/app/models/tag_alias.py b/backend/app/models/tag_alias.py index 93f4755..533cec3 100644 --- a/backend/app/models/tag_alias.py +++ b/backend/app/models/tag_alias.py @@ -5,7 +5,7 @@ in image_prediction stay unmolested. from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, String, func +from sqlalchemy import DateTime, ForeignKey, Index, String, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -14,10 +14,17 @@ from .base import Base class TagAlias(Base): __tablename__ = "tag_alias" + + __table_args__ = ( + # Named explicitly: the database calls this ix_tag_alias_canonical, while + # a bare index=True on the column would generate ix_tag_alias_canonical_tag_id + # and silently propose a drop+create on the next autogenerate (#3275). + Index("ix_tag_alias_canonical", "canonical_tag_id"), + ) alias_string: Mapped[str] = mapped_column(String(255), primary_key=True) alias_category: Mapped[str] = mapped_column(String(32), primary_key=True) canonical_tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/tag_suggestion_rejection.py b/backend/app/models/tag_suggestion_rejection.py index 8a834fe..02756de 100644 --- a/backend/app/models/tag_suggestion_rejection.py +++ b/backend/app/models/tag_suggestion_rejection.py @@ -5,7 +5,7 @@ Prevents re-suggestion AND prevents allowlist auto-apply on that image. from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, func +from sqlalchemy import DateTime, ForeignKey, Index, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -14,11 +14,16 @@ from .base import Base class TagSuggestionRejection(Base): __tablename__ = "tag_suggestion_rejection" + + __table_args__ = ( + # Named explicitly; see tag_alias for why (#3275). + Index("ix_tag_suggestion_rejection_tag", "tag_id"), + ) image_record_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True ) tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True ) rejected_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/task_run.py b/backend/app/models/task_run.py index e8eac05..c24ed46 100644 --- a/backend/app/models/task_run.py +++ b/backend/app/models/task_run.py @@ -15,7 +15,7 @@ backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min). from datetime import datetime -from sqlalchemy import DateTime, Integer, String, Text +from sqlalchemy import DateTime, Index, Integer, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -24,6 +24,13 @@ from .base import Base class TaskRun(Base): __tablename__ = "task_run" + + __table_args__ = ( + # alembic 0016: the three task-history indexes (#3275). + Index("ix_task_run_name_started", "task_name", text("started_at DESC")), + Index("ix_task_run_queue_started", "queue", text("started_at DESC")), + Index("ix_task_run_status_started", "status", text("started_at DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) celery_task_id: Mapped[str] = mapped_column( String(64), nullable=False, index=True, @@ -40,6 +47,7 @@ class TaskRun(Base): duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) status: Mapped[str] = mapped_column( String(16), nullable=False, default="running", index=True, + server_default="running", ) error_type: Mapped[str | None] = mapped_column(String(128), nullable=True) error_message: Mapped[str | None] = mapped_column(Text, nullable=True)