Post.source_id refactor + tick/backfill modes + PARTIAL classifier + Mux fix + UX #44
@@ -20,6 +20,7 @@ from datetime import datetime
|
|||||||
|
|
||||||
from sqlalchemy import Select, and_, exists, func, or_, select
|
from sqlalchemy import Select, and_, exists, func, or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from sqlalchemy.orm import aliased
|
||||||
|
|
||||||
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
@@ -139,10 +140,17 @@ def _provenance_clause(post_id, artist_id):
|
|||||||
# path now drops NULL-source provenance rows (filesystem-imported
|
# path now drops NULL-source provenance rows (filesystem-imported
|
||||||
# content) which would otherwise vanish from artist-filtered
|
# content) which would otherwise vanish from artist-filtered
|
||||||
# gallery views.
|
# gallery views.
|
||||||
|
# ALIAS Post: the gallery query outer-joins Post on
|
||||||
|
# ImageRecord.primary_post_id (`_outer_join_primary_post`).
|
||||||
|
# SQLAlchemy would otherwise correlate a bare `Post` reference
|
||||||
|
# in this EXISTS subquery to that outer Post (which is NULL for
|
||||||
|
# images with no primary post), and the filter would silently
|
||||||
|
# match nothing.
|
||||||
|
post_inner = aliased(Post)
|
||||||
return exists().where(
|
return exists().where(
|
||||||
ImageProvenance.image_record_id == ImageRecord.id,
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
ImageProvenance.post_id == Post.id,
|
ImageProvenance.post_id == post_inner.id,
|
||||||
Post.artist_id == artist_id,
|
post_inner.artist_id == artist_id,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ from backend.app.models import (
|
|||||||
ImageProvenance,
|
ImageProvenance,
|
||||||
ImageRecord,
|
ImageRecord,
|
||||||
ImportSettings,
|
ImportSettings,
|
||||||
Source,
|
|
||||||
)
|
)
|
||||||
from backend.app.services.importer import Importer
|
from backend.app.services.importer import Importer
|
||||||
from backend.app.services.thumbnailer import Thumbnailer
|
from backend.app.services.thumbnailer import Thumbnailer
|
||||||
@@ -139,9 +138,11 @@ def test_apply_sidecar_recovers_from_integrity_error(
|
|||||||
r = importer.import_one(m)
|
r = importer.import_one(m)
|
||||||
assert r.status == "imported"
|
assert r.status == "imported"
|
||||||
rec = importer.session.get(ImageRecord, r.image_id)
|
rec = importer.session.get(ImageRecord, r.image_id)
|
||||||
src = importer.session.execute(select(Source)).scalar_one()
|
# alembic 0030 stopped creating synthetic Source rows for filesystem
|
||||||
|
# sidecars; the Post sits null-source and the provenance row points at
|
||||||
|
# it directly. The race-recovery path tested below operates on
|
||||||
|
# ImageProvenance regardless of Source presence.
|
||||||
assert rec is not None
|
assert rec is not None
|
||||||
assert src is not None
|
|
||||||
|
|
||||||
# Monkeypatch session.execute so the FIRST select inside _apply_sidecar's
|
# Monkeypatch session.execute so the FIRST select inside _apply_sidecar's
|
||||||
# existence-check returns a "no row" wrapper. Subsequent selects (e.g.
|
# existence-check returns a "no row" wrapper. Subsequent selects (e.g.
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ from backend.app.models import (
|
|||||||
ImageRecord,
|
ImageRecord,
|
||||||
ImportSettings,
|
ImportSettings,
|
||||||
Post,
|
Post,
|
||||||
Source,
|
|
||||||
Tag,
|
Tag,
|
||||||
TagKind,
|
TagKind,
|
||||||
)
|
)
|
||||||
@@ -197,10 +196,11 @@ def test_supersede_applies_new_file_sidecar(importer, import_layout):
|
|||||||
assert post.post_title == "Set 1"
|
assert post.post_title == "Set 1"
|
||||||
assert "big version" in (post.description or "")
|
assert "big version" in (post.description or "")
|
||||||
|
|
||||||
source = importer.session.execute(
|
# Filesystem sidecars no longer create a synthetic Source (alembic 0030).
|
||||||
select(Source).where(Source.id == post.source_id)
|
# The Post sits null-source; the platform context is captured in the
|
||||||
).scalar_one()
|
# sidecar JSON / raw_metadata, not in a phantom Source row.
|
||||||
assert source.platform == "patreon"
|
assert post.source_id is None
|
||||||
|
assert post.artist_id is not None
|
||||||
|
|
||||||
prov_count = importer.session.execute(
|
prov_count = importer.session.execute(
|
||||||
select(func.count(ImageProvenance.id))
|
select(func.count(ImageProvenance.id))
|
||||||
|
|||||||
@@ -75,9 +75,15 @@ def test_sidecar_creates_provenance(importer, import_layout):
|
|||||||
r = importer.import_one(m)
|
r = importer.import_one(m)
|
||||||
assert r.status == "imported"
|
assert r.status == "imported"
|
||||||
rec = importer.session.get(ImageRecord, r.image_id)
|
rec = importer.session.get(ImageRecord, r.image_id)
|
||||||
src = importer.session.execute(select(Source)).scalar_one()
|
|
||||||
post = importer.session.execute(select(Post)).scalar_one()
|
post = importer.session.execute(select(Post)).scalar_one()
|
||||||
assert src.platform == "patreon"
|
# Filesystem-imported sidecar posts no longer create a synthetic Source
|
||||||
|
# (alembic 0030 / nullable post.source_id refactor). The Post is linked
|
||||||
|
# to the artist via Post.artist_id; Post.source_id stays NULL until a
|
||||||
|
# real subscription for the (artist, platform) gets added.
|
||||||
|
assert post.source_id is None
|
||||||
|
assert importer.session.execute(
|
||||||
|
select(func.count()).select_from(Source)
|
||||||
|
).scalar_one() == 0
|
||||||
assert post.external_post_id == "555"
|
assert post.external_post_id == "555"
|
||||||
assert post.post_url == "https://patreon.com/posts/555"
|
assert post.post_url == "https://patreon.com/posts/555"
|
||||||
assert post.post_title == "Set 1"
|
assert post.post_title == "Set 1"
|
||||||
@@ -106,9 +112,11 @@ def test_reimport_same_post_idempotent(importer, import_layout):
|
|||||||
_sidecar(m2, payload)
|
_sidecar(m2, payload)
|
||||||
r2 = importer.import_one(m2)
|
r2 = importer.import_one(m2)
|
||||||
assert r2.status == "imported"
|
assert r2.status == "imported"
|
||||||
|
# No synthetic Source after alembic 0030; both imports still resolve to
|
||||||
|
# a single null-source Post (deduped by uq_post_artist_external_id_null_source).
|
||||||
assert importer.session.execute(
|
assert importer.session.execute(
|
||||||
select(func.count()).select_from(Source)
|
select(func.count()).select_from(Source)
|
||||||
).scalar_one() == 1
|
).scalar_one() == 0
|
||||||
assert importer.session.execute(
|
assert importer.session.execute(
|
||||||
select(func.count()).select_from(Post)
|
select(func.count()).select_from(Post)
|
||||||
).scalar_one() == 1
|
).scalar_one() == 1
|
||||||
@@ -164,5 +172,11 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
|
|||||||
a = importer.session.execute(
|
a = importer.session.execute(
|
||||||
select(Artist).where(Artist.slug == "yuki")
|
select(Artist).where(Artist.slug == "yuki")
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
src = importer.session.execute(select(Source)).scalar_one()
|
# No synthetic Source after alembic 0030; the artist linkage lives on
|
||||||
assert src.artist_id == a.id
|
# Post.artist_id (NOT NULL FK).
|
||||||
|
post = importer.session.execute(select(Post)).scalar_one()
|
||||||
|
assert post.artist_id == a.id
|
||||||
|
assert post.source_id is None
|
||||||
|
assert importer.session.execute(
|
||||||
|
select(func.count()).select_from(Source)
|
||||||
|
).scalar_one() == 0
|
||||||
|
|||||||
Reference in New Issue
Block a user