feat(tags): legacy-tag purge also catches source:* general tags
Operator-confirmed 2026-05-28: the BlenderKnight:* tags are `archive`
kind (caught by the kind purge), but the `source:patreon`-style tags
are IR's old `source` kind that fell back to `general` during migration
(FC's enum has no `source` kind) — so they can't be matched by kind.
Broadened the purge to a two-rule match and renamed it for accuracy
(all dev-only, unreleased):
- cleanup_service.purge_tags_by_kind → purge_legacy_tags. Predicate is
now `kind IN (archive, post, artist) OR name LIKE 'source:%'`
(LEGACY_NAME_PREFIXES). Preview classifies each row into by_kind OR
by_prefix (source:* counts once under the prefix bucket regardless of
its general kind).
- endpoint /tags/purge-retired-kinds → /tags/purge-legacy. dry-run
returns by_kind + by_prefix + count + sample.
- store purgeRetiredKindTags → purgeLegacyTags.
- Tag Maintenance card copy + breakdown updated to show both buckets;
button reads "Preview/Delete legacy tags".
Tests updated + extended: dry-run reports by_kind {archive,post,artist}
AND by_prefix {source:*}, plain general/character tags survive; commit
deletes both the kind-matched and source:*-matched rows and leaves the
rest.
This commit is contained in:
@@ -6,6 +6,7 @@ Five action surfaces:
|
|||||||
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||||
POST /api/admin/tags/prune-unused (Tier A)
|
POST /api/admin/tags/prune-unused (Tier A)
|
||||||
|
POST /api/admin/tags/purge-legacy (Tier A)
|
||||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||||
|
|
||||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||||
@@ -208,20 +209,21 @@ async def tags_prune_unused():
|
|||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
@admin_bp.route("/tags/purge-retired-kinds", methods=["POST"])
|
@admin_bp.route("/tags/purge-legacy", methods=["POST"])
|
||||||
async def tags_purge_retired_kinds():
|
async def tags_purge_legacy():
|
||||||
"""Tier-A: delete tags whose kind is a system/legacy kind FC no
|
"""Tier-A: delete legacy IR-migration tags — archive/post/artist
|
||||||
longer creates (archive / post / artist — the IR-migration leftovers
|
kinds (e.g. `BlenderKnight:Hannah_BJ_Loops`) PLUS general tags with
|
||||||
like `BlenderKnight:Hannah_BJ_Loops`). dry-run preview returns
|
a legacy name prefix (`source:*`, from IR's source kind that fell
|
||||||
per-kind counts + a sample so the UI can show exactly what'll go
|
back to general). dry-run preview returns per-kind + per-prefix
|
||||||
before the operator confirms with dry_run=false."""
|
counts + a sample so the UI shows exactly what'll go before the
|
||||||
from ..services.cleanup_service import purge_tags_by_kind
|
operator confirms with dry_run=false."""
|
||||||
|
from ..services.cleanup_service import purge_legacy_tags
|
||||||
|
|
||||||
body = await request.get_json(silent=True) or {}
|
body = await request.get_json(silent=True) or {}
|
||||||
dry_run = bool(body.get("dry_run", False))
|
dry_run = bool(body.get("dry_run", False))
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
result = await session.run_sync(
|
result = await session.run_sync(
|
||||||
lambda sync_sess: purge_tags_by_kind(sync_sess, dry_run=dry_run)
|
lambda sync_sess: purge_legacy_tags(sync_sess, dry_run=dry_run)
|
||||||
)
|
)
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from datetime import UTC, datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, or_, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||||
@@ -369,46 +369,70 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
|||||||
return {"deleted": len(ids), "sample_names": sample}
|
return {"deleted": len(ids), "sample_names": sample}
|
||||||
|
|
||||||
|
|
||||||
# System/legacy tag kinds that FC no longer creates: the tag input only
|
# Legacy tags FC no longer uses, in two shapes:
|
||||||
# makes character/fandom/series/general; provenance (post grouping) and
|
# (1) kinds the tag input never produces — archive/post/artist.
|
||||||
# archive membership are their own systems now, and artists are
|
# provenance (post grouping) + archive membership are their own
|
||||||
# first-class Artist/Source rows. These linger only on IR-migrated rows.
|
# systems now, and artists are first-class Artist/Source rows.
|
||||||
# meta/rating were hard-deleted by alembic 0023, so they're not listed.
|
# meta/rating were already hard-deleted by alembic 0023.
|
||||||
|
# (2) name prefixes from IR kinds FC never adopted — `source:*`.
|
||||||
|
# ImageRepo had a `source` kind; FC's enum doesn't, so ir_ingest
|
||||||
|
# fell those back to `general` (kind=general, name="source:patreon"
|
||||||
|
# etc.). They can't be caught by kind, so we match the name prefix.
|
||||||
PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
|
PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
|
||||||
|
LEGACY_NAME_PREFIXES = ("source:",)
|
||||||
|
|
||||||
|
|
||||||
def purge_tags_by_kind(
|
def _legacy_tag_predicate():
|
||||||
session: Session, *, kinds: tuple[str, ...] = PURGEABLE_TAG_KINDS,
|
name_clauses = [Tag.name.like(f"{p}%") for p in LEGACY_NAME_PREFIXES]
|
||||||
dry_run: bool = False,
|
return or_(Tag.kind.in_(PURGEABLE_TAG_KINDS), *name_clauses)
|
||||||
) -> dict:
|
|
||||||
"""Count (dry_run) or delete all tags whose kind is in `kinds`.
|
|
||||||
|
def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||||
|
"""Count (dry_run) or delete legacy IR-migration tags: archive/post/
|
||||||
|
artist-kind tags PLUS general tags whose name matches a legacy
|
||||||
|
prefix (source:*).
|
||||||
|
|
||||||
CASCADE on image_tag / tag_alias / tag_allowlist /
|
CASCADE on image_tag / tag_alias / tag_allowlist /
|
||||||
tag_reference_embedding / tag_suggestion_rejection / series_page
|
tag_reference_embedding / tag_suggestion_rejection / series_page
|
||||||
clears the related rows on the parent DELETE.
|
clears the related rows on the parent DELETE.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
{"by_kind": {kind: count, ...}, "count": total,
|
{"by_kind": {kind: count, ...}, # kind-matched rows
|
||||||
"sample_names": [first 50], and on live runs "deleted": total}
|
"by_prefix": {"source:*": count}, # name-prefix-matched rows
|
||||||
|
"count": total, "sample_names": [first 50],
|
||||||
|
and on live runs "deleted": total}
|
||||||
"""
|
"""
|
||||||
|
predicate = _legacy_tag_predicate()
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(Tag.id, Tag.name, Tag.kind).where(Tag.kind.in_(kinds))
|
select(Tag.id, Tag.name, Tag.kind).where(predicate)
|
||||||
).all()
|
).all()
|
||||||
by_kind: dict[str, int] = {}
|
by_kind: dict[str, int] = {}
|
||||||
for _id, _name, kind in rows:
|
by_prefix: dict[str, int] = {}
|
||||||
key = kind.value if hasattr(kind, "value") else str(kind)
|
for _id, name, kind in rows:
|
||||||
by_kind[key] = by_kind.get(key, 0) + 1
|
# Classify by name-prefix first so a source:* row counts once,
|
||||||
|
# under the prefix bucket, regardless of its (general) kind.
|
||||||
|
matched_prefix = next(
|
||||||
|
(p for p in LEGACY_NAME_PREFIXES if name.startswith(p)), None,
|
||||||
|
)
|
||||||
|
if matched_prefix is not None:
|
||||||
|
label = f"{matched_prefix}*"
|
||||||
|
by_prefix[label] = by_prefix.get(label, 0) + 1
|
||||||
|
else:
|
||||||
|
key = kind.value if hasattr(kind, "value") else str(kind)
|
||||||
|
by_kind[key] = by_kind.get(key, 0) + 1
|
||||||
sample = [name for _id, name, _kind in rows[:50]]
|
sample = [name for _id, name, _kind in rows[:50]]
|
||||||
total = len(rows)
|
total = len(rows)
|
||||||
if dry_run:
|
result = {
|
||||||
return {"by_kind": by_kind, "count": total, "sample_names": sample}
|
"by_kind": by_kind, "by_prefix": by_prefix,
|
||||||
if total:
|
"count": total, "sample_names": sample,
|
||||||
session.execute(Tag.__table__.delete().where(Tag.kind.in_(kinds)))
|
|
||||||
session.commit()
|
|
||||||
return {
|
|
||||||
"by_kind": by_kind, "count": total, "deleted": total,
|
|
||||||
"sample_names": sample,
|
|
||||||
}
|
}
|
||||||
|
if dry_run:
|
||||||
|
return result
|
||||||
|
if total:
|
||||||
|
session.execute(Tag.__table__.delete().where(predicate))
|
||||||
|
session.commit()
|
||||||
|
result["deleted"] = total
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -48,12 +48,14 @@
|
|||||||
<v-divider class="my-5" />
|
<v-divider class="my-5" />
|
||||||
|
|
||||||
<p class="fc-muted text-body-2 mb-4">
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
Purge tags with retired/system kinds FC no longer uses
|
Purge legacy IR-migration tags FC no longer uses: retired/system
|
||||||
(<code>archive</code>, <code>post</code>, <code>artist</code>) —
|
kinds (<code>archive</code>, <code>post</code>,
|
||||||
IR-migration leftovers like
|
<code>artist</code> — e.g.
|
||||||
<code>BlenderKnight:Hannah_BJ_Loops</code>. Provenance and
|
<code>BlenderKnight:Hannah_BJ_Loops</code>) plus
|
||||||
artists are their own systems now, so these tag rows are pure
|
<code>source:*</code> tags (ImageRepo's old <code>source</code>
|
||||||
noise. Removes them from every image.
|
kind, which migrated to <code>general</code>). Provenance and
|
||||||
|
artists are their own systems now, so these are pure noise.
|
||||||
|
Removes them from every image.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<v-btn
|
<v-btn
|
||||||
@@ -62,14 +64,17 @@
|
|||||||
:loading="loadingKindPreview"
|
:loading="loadingKindPreview"
|
||||||
class="mb-3"
|
class="mb-3"
|
||||||
@click="onKindPreview"
|
@click="onKindPreview"
|
||||||
>Preview retired-kind tags</v-btn>
|
>Preview legacy tags</v-btn>
|
||||||
|
|
||||||
<div v-if="kindPreview">
|
<div v-if="kindPreview">
|
||||||
<p class="text-body-2 mb-2">
|
<p class="text-body-2 mb-2">
|
||||||
<strong>{{ kindPreview.count }}</strong> tag(s) of retired kinds.
|
<strong>{{ kindPreview.count }}</strong> legacy tag(s).
|
||||||
<span v-for="(n, k) in kindPreview.by_kind" :key="k" class="fc-muted">
|
<span v-for="(n, k) in kindPreview.by_kind" :key="k" class="fc-muted">
|
||||||
{{ k }}: {{ n }}
|
{{ k }}: {{ n }}
|
||||||
</span>
|
</span>
|
||||||
|
<span v-for="(n, p) in kindPreview.by_prefix" :key="p" class="fc-muted">
|
||||||
|
{{ p }}: {{ n }}
|
||||||
|
</span>
|
||||||
</p>
|
</p>
|
||||||
<div v-if="kindPreview.sample_names?.length" class="fc-name-grid mb-3">
|
<div v-if="kindPreview.sample_names?.length" class="fc-name-grid mb-3">
|
||||||
<span v-for="n in kindPreview.sample_names" :key="n" class="fc-name">
|
<span v-for="n in kindPreview.sample_names" :key="n" class="fc-name">
|
||||||
@@ -82,7 +87,7 @@
|
|||||||
:disabled="!kindPreview.count"
|
:disabled="!kindPreview.count"
|
||||||
:loading="kindCommitting"
|
:loading="kindCommitting"
|
||||||
@click="onKindCommit"
|
@click="onKindCommit"
|
||||||
>Delete {{ kindPreview.count }} retired-kind tag(s)</v-btn>
|
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
|
||||||
</div>
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
@@ -123,7 +128,7 @@ async function onCommit() {
|
|||||||
async function onKindPreview() {
|
async function onKindPreview() {
|
||||||
loadingKindPreview.value = true
|
loadingKindPreview.value = true
|
||||||
try {
|
try {
|
||||||
kindPreview.value = await store.purgeRetiredKindTags({ dryRun: true })
|
kindPreview.value = await store.purgeLegacyTags({ dryRun: true })
|
||||||
} finally {
|
} finally {
|
||||||
loadingKindPreview.value = false
|
loadingKindPreview.value = false
|
||||||
}
|
}
|
||||||
@@ -132,8 +137,8 @@ async function onKindPreview() {
|
|||||||
async function onKindCommit() {
|
async function onKindCommit() {
|
||||||
kindCommitting.value = true
|
kindCommitting.value = true
|
||||||
try {
|
try {
|
||||||
await store.purgeRetiredKindTags({ dryRun: false })
|
await store.purgeLegacyTags({ dryRun: false })
|
||||||
kindPreview.value = { count: 0, by_kind: {}, sample_names: [] }
|
kindPreview.value = { count: 0, by_kind: {}, by_prefix: {}, sample_names: [] }
|
||||||
} finally {
|
} finally {
|
||||||
kindCommitting.value = false
|
kindCommitting.value = false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,11 +114,11 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function purgeRetiredKindTags({ dryRun = true } = {}) {
|
async function purgeLegacyTags({ dryRun = true } = {}) {
|
||||||
lastError.value = null
|
lastError.value = null
|
||||||
try {
|
try {
|
||||||
return await api.post(
|
return await api.post(
|
||||||
'/api/admin/tags/purge-retired-kinds',
|
'/api/admin/tags/purge-legacy',
|
||||||
{ body: { dry_run: dryRun } },
|
{ body: { dry_run: dryRun } },
|
||||||
)
|
)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -161,7 +161,7 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
mergeTags,
|
mergeTags,
|
||||||
tagUsageCount,
|
tagUsageCount,
|
||||||
pruneUnusedTags,
|
pruneUnusedTags,
|
||||||
purgeRetiredKindTags,
|
purgeLegacyTags,
|
||||||
pollTaskUntilDone,
|
pollTaskUntilDone,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
+24
-17
@@ -367,50 +367,57 @@ async def test_prune_unused_commit_deletes_and_returns_count(client, db):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_purge_retired_kinds_dry_run_counts_by_kind(client, db):
|
async def test_purge_legacy_dry_run_counts_by_kind_and_prefix(client, db):
|
||||||
db.add_all([
|
db.add_all([
|
||||||
Tag(name="BlenderKnight:Hannah_BJ_Loops", kind=TagKind.archive),
|
Tag(name="BlenderKnight:Hannah_BJ_Loops", kind=TagKind.archive),
|
||||||
Tag(name="BlenderKnight:May Animation", kind=TagKind.post),
|
Tag(name="BlenderKnight:May Animation", kind=TagKind.post),
|
||||||
Tag(name="SomeArtist", kind=TagKind.artist),
|
Tag(name="SomeArtist", kind=TagKind.artist),
|
||||||
|
# IR `source` kind fell back to general during migration —
|
||||||
|
# caught by the source:* name prefix, not by kind.
|
||||||
|
Tag(name="source:patreon", kind=TagKind.general),
|
||||||
Tag(name="blonde hair", kind=TagKind.general), # must survive
|
Tag(name="blonde hair", kind=TagKind.general), # must survive
|
||||||
])
|
])
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
resp = await client.post(
|
resp = await client.post(
|
||||||
"/api/admin/tags/purge-retired-kinds", json={"dry_run": True},
|
"/api/admin/tags/purge-legacy", json={"dry_run": True},
|
||||||
)
|
)
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["count"] == 3
|
assert body["count"] == 4
|
||||||
assert body["by_kind"] == {"archive": 1, "post": 1, "artist": 1}
|
assert body["by_kind"] == {"archive": 1, "post": 1, "artist": 1}
|
||||||
|
assert body["by_prefix"] == {"source:*": 1}
|
||||||
assert "blonde hair" not in body["sample_names"]
|
assert "blonde hair" not in body["sample_names"]
|
||||||
|
assert "source:patreon" in body["sample_names"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_purge_retired_kinds_commit_deletes_only_retired(client, db):
|
async def test_purge_legacy_commit_deletes_only_legacy(client, db):
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
db.add_all([
|
db.add_all([
|
||||||
Tag(name="arch1", kind=TagKind.archive),
|
Tag(name="arch1", kind=TagKind.archive),
|
||||||
Tag(name="post1", kind=TagKind.post),
|
Tag(name="source:fanbox", kind=TagKind.general),
|
||||||
Tag(name="keepme", kind=TagKind.general),
|
Tag(name="keepme", kind=TagKind.general),
|
||||||
Tag(name="char1", kind=TagKind.character),
|
Tag(name="char1", kind=TagKind.character),
|
||||||
])
|
])
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
resp = await client.post(
|
resp = await client.post(
|
||||||
"/api/admin/tags/purge-retired-kinds", json={"dry_run": False},
|
"/api/admin/tags/purge-legacy", json={"dry_run": False},
|
||||||
)
|
)
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["deleted"] == 2
|
assert body["deleted"] == 2 # arch1 + source:fanbox
|
||||||
|
|
||||||
# general + character tags survive.
|
# The plain general + character tags survive.
|
||||||
surviving = (await db.execute(
|
keep = (await db.execute(
|
||||||
select(func.count()).select_from(Tag)
|
select(Tag.name).where(Tag.name.in_(["keepme", "char1"]))
|
||||||
.where(Tag.kind.in_([TagKind.general, TagKind.character]))
|
)).scalars().all()
|
||||||
|
assert set(keep) == {"keepme", "char1"}
|
||||||
|
# No archive/post/artist or source:* left.
|
||||||
|
gone = (await db.execute(
|
||||||
|
select(func.count()).select_from(Tag).where(
|
||||||
|
(Tag.kind.in_([TagKind.archive, TagKind.post, TagKind.artist]))
|
||||||
|
| (Tag.name.like("source:%"))
|
||||||
|
)
|
||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
assert surviving >= 2
|
assert gone == 0
|
||||||
retired_left = (await db.execute(
|
|
||||||
select(func.count()).select_from(Tag)
|
|
||||||
.where(Tag.kind.in_([TagKind.archive, TagKind.post, TagKind.artist]))
|
|
||||||
)).scalar_one()
|
|
||||||
assert retired_left == 0
|
|
||||||
|
|||||||
Reference in New Issue
Block a user