feat(fc3c): migration 0013 — download_event.metadata + import_settings downloader fields
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
|||||||
|
"""fc3c: download_event.metadata + import_settings downloader fields
|
||||||
|
|
||||||
|
Revision ID: 0013
|
||||||
|
Revises: 0012
|
||||||
|
Create Date: 2026-05-20
|
||||||
|
|
||||||
|
Additive only. download_event.metadata is the rich JSONB blob FC-3c
|
||||||
|
populates per run (run_stats, stdout/stderr, quarantined paths, import
|
||||||
|
summary). import_settings gains two operator-tunable downloader knobs:
|
||||||
|
download_rate_limit_seconds (gallery-dl extractor.sleep) and
|
||||||
|
download_validate_files (toggle the magic-byte validator).
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0013"
|
||||||
|
down_revision: Union[str, None] = "0012"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"download_event",
|
||||||
|
sa.Column(
|
||||||
|
"metadata", postgresql.JSONB,
|
||||||
|
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"download_rate_limit_seconds", sa.Float(),
|
||||||
|
nullable=False, server_default="3.0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"download_validate_files", sa.Boolean(),
|
||||||
|
nullable=False, server_default=sa.true(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_settings", "download_validate_files")
|
||||||
|
op.drop_column("import_settings", "download_rate_limit_seconds")
|
||||||
|
op.drop_column("download_event", "metadata")
|
||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -18,7 +20,7 @@ class DownloadEvent(Base):
|
|||||||
post_id: Mapped[int | None] = mapped_column(
|
post_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
)
|
)
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error
|
status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error|skipped
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
@@ -26,3 +28,7 @@ class DownloadEvent(Base):
|
|||||||
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||||
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
metadata_: Mapped[dict] = mapped_column(
|
||||||
|
"metadata", JSONB, nullable=False, default=dict,
|
||||||
|
server_default=sa.text("'{}'::jsonb"),
|
||||||
|
)
|
||||||
|
|||||||
@@ -30,3 +30,11 @@ class ImportSettings(Base):
|
|||||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
||||||
|
|
||||||
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
|
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
|
||||||
|
|
||||||
|
# FC-3c downloader knobs
|
||||||
|
download_rate_limit_seconds: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=3.0
|
||||||
|
)
|
||||||
|
download_validate_files: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=True
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import pytest
|
||||||
|
from sqlalchemy import inspect, select
|
||||||
|
|
||||||
|
from backend.app.models import ImportSettings
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_event_has_metadata(db):
|
||||||
|
cols = await db.run_sync(
|
||||||
|
lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("download_event")}
|
||||||
|
)
|
||||||
|
assert "metadata" in cols
|
||||||
|
assert cols["metadata"]["nullable"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_import_settings_has_downloader_fields(db):
|
||||||
|
cols = await db.run_sync(
|
||||||
|
lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("import_settings")}
|
||||||
|
)
|
||||||
|
assert "download_rate_limit_seconds" in cols
|
||||||
|
assert "download_validate_files" in cols
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_import_settings_defaults(db):
|
||||||
|
row = (await db.execute(select(ImportSettings).where(ImportSettings.id == 1))).scalar_one()
|
||||||
|
assert row.download_rate_limit_seconds == 3.0
|
||||||
|
assert row.download_validate_files is True
|
||||||
Reference in New Issue
Block a user