refactor(I5): remove one-and-done GS/IR migration tooling
The GS/IR migration cutover is complete, so the runbook tooling is dead weight. Removed: - services/migrators/ (gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup), tasks/migration.py, api/migrate.py (+ blueprint registration) - MigrationRun model; alembic 0027 drops the migration_run table - frontend LegacyMigrationCard + migration store (+ MaintenancePanel ref) - celery include + task route + celery_signals queue mapping for migration.* - the 1 GB MAX_CONTENT_LENGTH / MAX_FORM_MEMORY override (added solely for the ir_ingest upload) - migration-surface tests (test_api_migrate, test_migration_verify, test_ir_ingest, test_gs_ingest, test_tag_apply) Kept: the alembic schema-migration tests (test_migration_00XX — unrelated) and cleanup_service.py (the permanent artist-cascade/unlink home). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,98 +0,0 @@
|
||||
"""FC-5: /api/migrate API tests."""
|
||||
import io
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from werkzeug.datastructures import FileStorage
|
||||
|
||||
import backend.app.tasks.migration # noqa: F401 — register celery task
|
||||
from backend.app.celery_app import celery
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _file_storage(payload: dict, filename: str = "export.json") -> FileStorage:
|
||||
return FileStorage(
|
||||
stream=io.BytesIO(json.dumps(payload).encode("utf-8")),
|
||||
filename=filename,
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
|
||||
# Retired 2026-05-24 (FC-3h): `test_post_backup_returns_202_and_id`
|
||||
# asserted the /api/migrate/backup endpoint, which was retired in FC-3h.
|
||||
# Backup is now a first-class feature at /api/system/backup/*;
|
||||
# coverage lives in tests/test_api_system_backup.py.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_rejects_unknown_kind(client):
|
||||
resp = await client.post("/api/migrate/destroy", json={})
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "unknown_kind"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_ingest_rejects_missing_file(client):
|
||||
resp = await client.post("/api/migrate/gs_ingest", form={})
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "missing_export_file"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate.run_migration",
|
||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
||||
)
|
||||
|
||||
export = {
|
||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
||||
"subscriptions": [], "credentials": [],
|
||||
}
|
||||
resp = await client.post(
|
||||
"/api/migrate/gs_ingest",
|
||||
form={"dry_run": "false"},
|
||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
|
||||
|
||||
# Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
|
||||
# the migration backup-gate rejected applies without a recent backup.
|
||||
# That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
|
||||
# FC-3h will own backup as a first-class feature) so the test was
|
||||
# testing dead behavior. Removed rather than rewritten.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_dry_run_allowed_without_backup(client, monkeypatch):
|
||||
# FC-3h: the _has_recent_pre_migration_backup gate was retired; dry-run
|
||||
# ingests were always allowed and now non-dry-run ingests are too.
|
||||
# This test stays as regression coverage that dry_run ingest still works.
|
||||
monkeypatch.setattr(
|
||||
"backend.app.api.migrate.run_migration",
|
||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
||||
)
|
||||
export = {
|
||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
||||
"subscriptions": [], "credentials": [],
|
||||
}
|
||||
resp = await client.post(
|
||||
"/api/migrate/gs_ingest",
|
||||
form={"dry_run": "true"},
|
||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_run_404_unknown(client):
|
||||
resp = await client.get("/api/migrate/runs/9999999")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_run_migration_task_registered():
|
||||
assert "backend.app.tasks.migration.run_migration" in celery.tasks
|
||||
@@ -1,122 +0,0 @@
|
||||
"""FC-5: gs_ingest unit tests."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import Artist, Credential, Source
|
||||
from backend.app.services.credential_crypto import CredentialCrypto
|
||||
from backend.app.services.migrators import gs_ingest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _gs_export(subscriptions=None, credentials=None) -> dict:
|
||||
return {
|
||||
"source_app": "gallerysubscriber",
|
||||
"schema_version": 1,
|
||||
"exported_at": datetime.now(UTC).isoformat(),
|
||||
"subscriptions": subscriptions or [],
|
||||
"credentials": credentials or [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscription_creates_artist_with_slug(db):
|
||||
data = _gs_export(subscriptions=[
|
||||
{"name": "Alice Author", "enabled": True, "metadata": {}, "sources": []},
|
||||
])
|
||||
counts = await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
||||
assert counts["rows_inserted"] >= 1
|
||||
|
||||
artist = (await db.execute(
|
||||
select(Artist).where(Artist.name == "Alice Author")
|
||||
)).scalar_one()
|
||||
assert artist.slug == "alice-author"
|
||||
assert artist.is_subscription is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_source_attached_to_artist(db):
|
||||
data = _gs_export(subscriptions=[{
|
||||
"name": "bob-author",
|
||||
"enabled": True,
|
||||
"metadata": {},
|
||||
"sources": [{
|
||||
"platform": "patreon",
|
||||
"url": "https://patreon.com/bob",
|
||||
"enabled": True,
|
||||
"check_interval": 28800,
|
||||
"metadata": {"k": "v"},
|
||||
}],
|
||||
}])
|
||||
await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
||||
|
||||
src = (await db.execute(
|
||||
select(Source).where(Source.url == "https://patreon.com/bob")
|
||||
)).scalar_one()
|
||||
artist = (await db.execute(
|
||||
select(Artist).where(Artist.id == src.artist_id)
|
||||
)).scalar_one()
|
||||
assert artist.name == "bob-author"
|
||||
assert src.platform == "patreon"
|
||||
assert src.check_interval_override == 28800
|
||||
assert src.config_overrides == {"k": "v"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credential_re_encrypted_with_fc_key(db, tmp_path):
|
||||
fc_key_path = tmp_path / "credential_key.b64"
|
||||
fc_crypto = CredentialCrypto(fc_key_path)
|
||||
|
||||
data = _gs_export(credentials=[{
|
||||
"platform": "patreon",
|
||||
"credential_type": "cookies",
|
||||
"plaintext": "session_cookie=abcdef",
|
||||
"expires_at": None,
|
||||
}])
|
||||
await gs_ingest.migrate_async(db, data=data, fc_crypto=fc_crypto, dry_run=False)
|
||||
|
||||
cred = (await db.execute(
|
||||
select(Credential).where(Credential.platform == "patreon")
|
||||
)).scalar_one()
|
||||
assert fc_crypto.decrypt(cred.encrypted_blob) == "session_cookie=abcdef"
|
||||
assert cred.credential_type == "cookies"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_makes_no_changes(db):
|
||||
data = _gs_export(subscriptions=[
|
||||
{"name": "dave", "enabled": True, "metadata": {}, "sources": []},
|
||||
])
|
||||
counts = await gs_ingest.migrate_async(db, data=data, dry_run=True)
|
||||
assert counts["rows_processed"] >= 1
|
||||
|
||||
after = (await db.execute(
|
||||
select(Artist).where(Artist.name == "dave")
|
||||
)).scalar_one_or_none()
|
||||
assert after is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_on_rerun(db):
|
||||
data = _gs_export(subscriptions=[
|
||||
{"name": "eve", "enabled": True, "metadata": {}, "sources": []},
|
||||
])
|
||||
await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
||||
counts2 = await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
||||
assert counts2["rows_skipped"] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_wrong_source_app(db):
|
||||
bad = {"source_app": "wrong", "schema_version": 1, "subscriptions": [], "credentials": []}
|
||||
with pytest.raises(ValueError):
|
||||
await gs_ingest.migrate_async(db, data=bad, dry_run=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_unsupported_schema_version(db):
|
||||
bad = {"source_app": "gallerysubscriber", "schema_version": 99, "subscriptions": [], "credentials": []}
|
||||
with pytest.raises(ValueError):
|
||||
await gs_ingest.migrate_async(db, data=bad, dry_run=False)
|
||||
@@ -1,129 +0,0 @@
|
||||
"""FC-5: ir_ingest unit tests."""
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import Tag
|
||||
from backend.app.services.migrators import ir_ingest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None):
|
||||
return {
|
||||
"source_app": "imagerepo",
|
||||
"schema_version": 1,
|
||||
"exported_at": datetime.now(UTC).isoformat(),
|
||||
"tags": tags or [],
|
||||
"image_artist_assignments": artist_assignments or [],
|
||||
"image_tag_associations": tag_associations or [],
|
||||
"series_pages": series_pages or [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_general_tag_created(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "blue_eyes", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
tag = (await db.execute(
|
||||
select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general")
|
||||
)).scalar_one()
|
||||
assert tag.fandom_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_character_tag_resolves_fandom_name(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "Wonderland", "kind": "fandom", "fandom_name": None},
|
||||
{"name": "Alice", "kind": "character", "fandom_name": "Wonderland"},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
fandom = (await db.execute(
|
||||
select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom")
|
||||
)).scalar_one()
|
||||
alice = (await db.execute(
|
||||
select(Tag).where(Tag.name == "Alice", Tag.kind == "character")
|
||||
)).scalar_one()
|
||||
assert alice.fandom_id == fandom.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_and_post_kinds_skipped(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "BobArtist", "kind": "artist", "fandom_name": None},
|
||||
{"name": "PostXYZ", "kind": "post", "fandom_name": None},
|
||||
{"name": "general_one", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
assert counts["rows_skipped"] >= 2
|
||||
|
||||
bob = (await db.execute(
|
||||
select(Tag).where(Tag.name == "BobArtist")
|
||||
)).scalar_one_or_none()
|
||||
post = (await db.execute(
|
||||
select(Tag).where(Tag.name == "PostXYZ")
|
||||
)).scalar_one_or_none()
|
||||
assert bob is None
|
||||
assert post is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_written_to_disk(db, tmp_path):
|
||||
data = _ir_export(
|
||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}],
|
||||
tag_associations=[
|
||||
{"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
|
||||
],
|
||||
series_pages=[
|
||||
{"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1},
|
||||
],
|
||||
)
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
||||
assert manifest_file.exists()
|
||||
manifest = json.loads(manifest_file.read_text())
|
||||
assert manifest["schema_version"] == 1
|
||||
assert len(manifest["image_artist_assignments"]) == 1
|
||||
assert len(manifest["image_tag_associations"]) == 1
|
||||
assert len(manifest["series_pages"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path):
|
||||
data = _ir_export(
|
||||
tags=[{"name": "dry", "kind": "general", "fandom_name": None}],
|
||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}],
|
||||
)
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True)
|
||||
|
||||
tag = (await db.execute(
|
||||
select(Tag).where(Tag.name == "dry")
|
||||
)).scalar_one_or_none()
|
||||
assert tag is None
|
||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
||||
assert not manifest_file.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_on_rerun(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "rerun", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
assert counts2["rows_skipped"] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_wrong_source_app(db, tmp_path):
|
||||
bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [],
|
||||
"image_artist_assignments": [], "image_tag_associations": [], "series_pages": []}
|
||||
with pytest.raises(ValueError):
|
||||
await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False)
|
||||
@@ -1,72 +0,0 @@
|
||||
"""FC-5: verify + ml_queue unit tests."""
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.models import Artist, ImageRecord
|
||||
from backend.app.services.migrators import ml_queue, verify
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_row_counts_match(db):
|
||||
db.add(Artist(name="vAlpha", slug="valpha", is_subscription=True))
|
||||
db.add(Artist(name="vBeta", slug="vbeta", is_subscription=False))
|
||||
await db.commit()
|
||||
result = await verify.verify_async(
|
||||
db, expected={"artist_subscriptions": 1},
|
||||
)
|
||||
assert result["artist_subscriptions"]["status"] == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_sha256_sample_matches_disk(db, tmp_path):
|
||||
f = tmp_path / "sample.bin"
|
||||
f.write_bytes(b"hello world")
|
||||
sha = hashlib.sha256(b"hello world").hexdigest()
|
||||
db.add(ImageRecord(
|
||||
path=str(f), sha256=sha, size_bytes=11, mime="application/octet-stream",
|
||||
origin="imported_filesystem",
|
||||
))
|
||||
await db.commit()
|
||||
result = await verify.verify_sha256_sample(db, sample_size=10)
|
||||
assert result["matched"] >= 1
|
||||
assert result["mismatched"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_sha256_sample_detects_missing_file(db, tmp_path):
|
||||
sha = "f" * 64
|
||||
db.add(ImageRecord(
|
||||
path=str(tmp_path / "does-not-exist.bin"),
|
||||
sha256=sha, size_bytes=1, mime="application/octet-stream",
|
||||
origin="imported_filesystem",
|
||||
))
|
||||
await db.commit()
|
||||
result = await verify.verify_sha256_sample(db, sample_size=10)
|
||||
assert result["missing"] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ml_queue_returns_count_for_unprocessed(db, tmp_path, monkeypatch):
|
||||
queued: list[int] = []
|
||||
|
||||
class _FakeTask:
|
||||
def delay(self, image_id):
|
||||
queued.append(image_id)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.ml.tag_and_embed", _FakeTask(),
|
||||
)
|
||||
|
||||
f = tmp_path / "q.bin"
|
||||
f.write_bytes(b"x")
|
||||
db.add(ImageRecord(
|
||||
path=str(f), sha256="q" * 64, size_bytes=1,
|
||||
mime="application/octet-stream", origin="imported_filesystem",
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
count = await ml_queue.queue_all_unprocessed_async(db)
|
||||
assert count >= 1
|
||||
@@ -1,395 +0,0 @@
|
||||
"""FC-5: tag_apply unit tests."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
SeriesPage,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
image_tag,
|
||||
)
|
||||
from backend.app.services.migrators import tag_apply
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
async def _seed_image(db, sha: str, *, suffix="x"):
|
||||
img = ImageRecord(
|
||||
path=f"/images/imported/aa/{suffix}.jpg",
|
||||
sha256=sha, size_bytes=10, mime="image/jpeg",
|
||||
width=10, height=10, origin="imported_filesystem",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
def _write_manifest(
|
||||
tmp_path, *,
|
||||
artist_assignments=None, tag_associations=None,
|
||||
series_pages=None, image_posts=None, schema_version=2,
|
||||
):
|
||||
d = tmp_path / "_migration_state"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "ir_tag_manifest.json").write_text(json.dumps({
|
||||
"schema_version": schema_version,
|
||||
"image_artist_assignments": artist_assignments or [],
|
||||
"image_tag_associations": tag_associations or [],
|
||||
"series_pages": series_pages or [],
|
||||
"image_posts": image_posts or [],
|
||||
}))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_assignment_sets_image_artist_id(db, tmp_path):
|
||||
sha = "a" * 64
|
||||
await _seed_image(db, sha)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, artist_assignments=[
|
||||
{"sha256": sha, "artist_name": "BobAuthor"},
|
||||
])
|
||||
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_inserted"] >= 1
|
||||
|
||||
img = (await db.execute(
|
||||
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
artist = (await db.execute(
|
||||
select(Artist).where(Artist.id == img.artist_id)
|
||||
)).scalar_one()
|
||||
assert artist.name == "BobAuthor"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_association_creates_image_tag(db, tmp_path):
|
||||
sha = "b" * 64
|
||||
await _seed_image(db, sha, suffix="b")
|
||||
# Seed the tag itself (ir_ingest would have created it).
|
||||
tag = Tag(name="blue_eyes", kind=TagKind.general)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, tag_associations=[
|
||||
{"sha256": sha, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
row = (await db.execute(
|
||||
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
|
||||
)).all()
|
||||
assert len(row) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unmatched_sha_appears_in_unmatched_list(db, tmp_path):
|
||||
_write_manifest(tmp_path, tag_associations=[
|
||||
{"sha256": "z" * 64, "tag_name": "missing_image", "tag_kind": "general", "fandom_name": None},
|
||||
])
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert any(e["sha256"] == "z" * 64 for e in result["unmatched"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_series_page_created(db, tmp_path):
|
||||
sha = "c" * 64
|
||||
await _seed_image(db, sha, suffix="c")
|
||||
series_tag = Tag(name="MySeries", kind=TagKind.series)
|
||||
db.add(series_tag)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, series_pages=[
|
||||
{"sha256": sha, "series_tag_name": "MySeries", "page_number": 1},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
sp = (await db.execute(
|
||||
select(SeriesPage).where(SeriesPage.image_id == img_id)
|
||||
)).scalar_one()
|
||||
assert sp.page_number == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_on_rerun(db, tmp_path):
|
||||
sha = "d" * 64
|
||||
await _seed_image(db, sha, suffix="d")
|
||||
tag = Tag(name="rerun_tag", kind=TagKind.general)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, tag_associations=[
|
||||
{"sha256": sha, "tag_name": "rerun_tag", "tag_kind": "general", "fandom_name": None},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result2["counts"]["rows_skipped"] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_manifest_raises(db, tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_makes_no_changes(db, tmp_path):
|
||||
sha = "e" * 64
|
||||
await _seed_image(db, sha, suffix="e")
|
||||
tag = Tag(name="dry_tag", kind=TagKind.general)
|
||||
db.add(tag)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, tag_associations=[
|
||||
{"sha256": sha, "tag_name": "dry_tag", "tag_kind": "general", "fandom_name": None},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
|
||||
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
rows = (await db.execute(
|
||||
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
|
||||
)).all()
|
||||
assert len(rows) == 0
|
||||
|
||||
|
||||
# --- Phase 4: image_posts (schema v2) -----------------------------------
|
||||
|
||||
|
||||
_POST_ENTRY = {
|
||||
"platform": "patreon",
|
||||
"post_id": "10001",
|
||||
"artist": "Maewix",
|
||||
"title": "Test Post Title",
|
||||
"description": "<p>HTML description</p>",
|
||||
"source_url": "https://www.patreon.com/posts/test-10001",
|
||||
"attachment_count": 1,
|
||||
"published_at": "2026-01-15T12:00:00+00:00",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_creates_source_post_provenance(db, tmp_path):
|
||||
sha = "d" * 64
|
||||
await _seed_image(db, sha, suffix="d")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_inserted"] >= 1
|
||||
|
||||
# Async Core-DML assertions — column selects, not ORM attribute access.
|
||||
artist_count = (await db.execute(
|
||||
select(func.count(Artist.id)).where(Artist.slug == "maewix")
|
||||
)).scalar_one()
|
||||
assert artist_count == 1
|
||||
|
||||
source_count = (await db.execute(
|
||||
select(func.count(Source.id)).where(
|
||||
Source.platform == "patreon",
|
||||
Source.url == "https://www.patreon.com/maewix",
|
||||
)
|
||||
)).scalar_one()
|
||||
assert source_count == 1
|
||||
|
||||
post_count = (await db.execute(
|
||||
select(func.count(Post.id)).where(Post.external_post_id == "10001")
|
||||
)).scalar_one()
|
||||
assert post_count == 1
|
||||
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
.where(ImageProvenance.image_record_id == img_id)
|
||||
)).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
# Phase 4 must also set ImageRecord.primary_post_id so the gallery's
|
||||
# effective_date COALESCE can surface Post.post_date. Operator-flagged
|
||||
# 2026-05-25: without this, IR-migrated images keep sorting by FC's
|
||||
# scan date instead of the original publish date.
|
||||
primary_post_id = (await db.execute(
|
||||
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
|
||||
)).scalar_one()
|
||||
canonical_post_id = (await db.execute(
|
||||
select(Post.id).where(Post.external_post_id == "10001")
|
||||
)).scalar_one()
|
||||
assert primary_post_id == canonical_post_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_primary_post_id_not_clobbered(db, tmp_path):
|
||||
"""If the importer already set primary_post_id (e.g. a downloaded
|
||||
image with a known provenance), phase 4 must NOT overwrite it when
|
||||
re-running tag_apply against the IR migration. The existing
|
||||
download-time linkage is the source of truth."""
|
||||
sha = "9" * 64
|
||||
await _seed_image(db, sha, suffix="9")
|
||||
# Pre-set primary_post_id to a sentinel Post so we can detect a clobber.
|
||||
img_id = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one()
|
||||
# Build an existing Source + Post for the sentinel.
|
||||
art = Artist(name="Pre-existing", slug="pre-existing")
|
||||
db.add(art)
|
||||
await db.flush()
|
||||
src = Source(
|
||||
artist_id=art.id, platform="patreon",
|
||||
url="https://www.patreon.com/pre-existing",
|
||||
)
|
||||
db.add(src)
|
||||
await db.flush()
|
||||
sentinel_post = Post(
|
||||
source_id=src.id, external_post_id="sentinel-99",
|
||||
post_title="Pre-existing",
|
||||
)
|
||||
db.add(sentinel_post)
|
||||
await db.flush()
|
||||
await db.execute(
|
||||
ImageRecord.__table__.update()
|
||||
.where(ImageRecord.id == img_id)
|
||||
.values(primary_post_id=sentinel_post.id)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
# The migration created a NEW Post (external_post_id="10001") and a
|
||||
# new ImageProvenance, but primary_post_id must still point at the
|
||||
# original sentinel.
|
||||
primary_post_id = (await db.execute(
|
||||
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
|
||||
)).scalar_one()
|
||||
assert primary_post_id == sentinel_post.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_idempotent_on_rerun(db, tmp_path):
|
||||
sha = "e" * 64
|
||||
await _seed_image(db, sha, suffix="e")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
|
||||
# Re-run: nothing should be newly inserted; all skipped.
|
||||
assert result2["counts"]["rows_inserted"] == 0
|
||||
assert result2["counts"]["rows_skipped"] >= 1
|
||||
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
)).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_unknown_sha_unmatched(db, tmp_path):
|
||||
# No image seeded; sha doesn't exist.
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": ["f" * 64]},
|
||||
])
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
unmatched_kinds = [u["kind"] for u in result["unmatched"]]
|
||||
assert "post" in unmatched_kinds
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_unknown_platform_skipped(db, tmp_path):
|
||||
sha = "1" * 64
|
||||
await _seed_image(db, sha, suffix="1")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "platform": "tumblr", "image_sha256s": [sha]},
|
||||
])
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_skipped"] >= 1
|
||||
|
||||
# No Post / Source / ImageProvenance should have been created.
|
||||
src_count = (await db.execute(
|
||||
select(func.count(Source.id)).where(Source.platform == "tumblr")
|
||||
)).scalar_one()
|
||||
assert src_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("platform,expected_url", [
|
||||
("deviantart", "https://www.deviantart.com/maewix"),
|
||||
("pixiv", "https://www.pixiv.net/users/maewix"),
|
||||
])
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_extended_platforms_create_source(
|
||||
db, tmp_path, platform, expected_url,
|
||||
):
|
||||
"""Regression for 2026-05-25 operator-reported bug: phase 4's
|
||||
_PLATFORM_PROFILE_URL had only patreon/subscribestar/hentaifoundry,
|
||||
silently dropping deviantart + pixiv PostMetadata from the IR migration."""
|
||||
sha = f"{platform[0]}" * 64
|
||||
await _seed_image(db, sha, suffix=f"_{platform}")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "platform": platform, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
||||
assert result["counts"]["rows_inserted"] >= 1
|
||||
|
||||
src_url = (await db.execute(
|
||||
select(Source.url).where(Source.platform == platform)
|
||||
)).scalar_one()
|
||||
assert src_url == expected_url
|
||||
|
||||
# ImageProvenance row was created.
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(Source.platform == platform)
|
||||
)).scalar_one()
|
||||
assert prov_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_posts_dry_run_makes_no_writes(db, tmp_path):
|
||||
sha = "2" * 64
|
||||
await _seed_image(db, sha, suffix="2")
|
||||
await db.commit()
|
||||
|
||||
_write_manifest(tmp_path, image_posts=[
|
||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
||||
])
|
||||
|
||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
|
||||
prov_count = (await db.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
)).scalar_one()
|
||||
assert prov_count == 0
|
||||
Reference in New Issue
Block a user