diff --git a/alembic/versions/0013_fc3c_download_event_metadata.py b/alembic/versions/0013_fc3c_download_event_metadata.py new file mode 100644 index 0000000..c88b3f9 --- /dev/null +++ b/alembic/versions/0013_fc3c_download_event_metadata.py @@ -0,0 +1,52 @@ +"""fc3c: download_event.metadata + import_settings downloader fields + +Revision ID: 0013 +Revises: 0012 +Create Date: 2026-05-20 + +Additive only. download_event.metadata is the rich JSONB blob FC-3c +populates per run (run_stats, stdout/stderr, quarantined paths, import +summary). import_settings gains two operator-tunable downloader knobs: +download_rate_limit_seconds (gallery-dl extractor.sleep) and +download_validate_files (toggle the magic-byte validator). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0013" +down_revision: Union[str, None] = "0012" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "download_event", + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_rate_limit_seconds", sa.Float(), + nullable=False, server_default="3.0", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_validate_files", sa.Boolean(), + nullable=False, server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "download_validate_files") + op.drop_column("import_settings", "download_rate_limit_seconds") + op.drop_column("download_event", "metadata") diff --git a/backend/app/models/download_event.py b/backend/app/models/download_event.py index 8955558..7fe5b37 100644 --- a/backend/app/models/download_event.py +++ b/backend/app/models/download_event.py @@ -2,7 +2,9 @@ from datetime import datetime +import sqlalchemy as sa from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -18,7 +20,7 @@ class DownloadEvent(Base): post_id: Mapped[int | None] = mapped_column( ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True ) - status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error + status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error|skipped started_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) @@ -26,3 +28,7 @@ class DownloadEvent(Base): bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) error: Mapped[str | None] = mapped_column(Text, nullable=True) + metadata_: Mapped[dict] = mapped_column( + "metadata", JSONB, nullable=False, default=dict, + server_default=sa.text("'{}'::jsonb"), + ) diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index be65d26..527f4ba 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -30,3 +30,11 @@ class ImportSettings(Base): single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30) phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + + # FC-3c downloader knobs + download_rate_limit_seconds: Mapped[float] = mapped_column( + Float, nullable=False, default=3.0 + ) + download_validate_files: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) diff --git a/tests/test_migration_0013.py b/tests/test_migration_0013.py new file mode 100644 index 0000000..1a18384 --- /dev/null +++ b/tests/test_migration_0013.py @@ -0,0 +1,31 @@ +import pytest +from sqlalchemy import inspect, select + +from backend.app.models import ImportSettings + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_download_event_has_metadata(db): + cols = await db.run_sync( + lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("download_event")} + ) + assert "metadata" in cols + assert cols["metadata"]["nullable"] is False + + +@pytest.mark.asyncio +async def test_import_settings_has_downloader_fields(db): + cols = await db.run_sync( + lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("import_settings")} + ) + assert "download_rate_limit_seconds" in cols + assert "download_validate_files" in cols + + +@pytest.mark.asyncio +async def test_import_settings_defaults(db): + row = (await db.execute(select(ImportSettings).where(ImportSettings.id == 1))).scalar_one() + assert row.download_rate_limit_seconds == 3.0 + assert row.download_validate_files is True