chore: retire ml-backfill-daily beat + the spent purge-legacy action (operator-approved)
- ml-backfill-daily: the CPU tag_and_embed backfill raced the GPU agent's daily embed backfill for the same NULL-embedding images at ~100x the cost (B1 audit verdict, milestone #124). The backfill TASK stays — the manual /api/ml/backfill button remains the deliberate CPU fallback pending B3. - purge-legacy: one-time IR-migration cleanup, dry-run verified 0 targets on the live library before removal (A2 audit, milestone #123). Fully retired per rule 22: tile, store action, route, service fn, tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -1,13 +1,12 @@
|
|||||||
"""FC-3k: /api/admin — destructive admin actions.
|
"""FC-3k: /api/admin — destructive admin actions.
|
||||||
|
|
||||||
Five action surfaces:
|
Action surfaces:
|
||||||
POST /api/admin/artists/<slug>/cascade-delete (Tier C)
|
POST /api/admin/artists/<slug>/cascade-delete (Tier C)
|
||||||
POST /api/admin/images/bulk-delete (Tier C)
|
POST /api/admin/images/bulk-delete (Tier C)
|
||||||
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||||
POST /api/admin/tags/prune-unused (Tier A)
|
POST /api/admin/tags/prune-unused (Tier A)
|
||||||
POST /api/admin/posts/prune-bare (Tier A)
|
POST /api/admin/posts/prune-bare (Tier A)
|
||||||
POST /api/admin/tags/purge-legacy (Tier A)
|
|
||||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||||
|
|
||||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||||
@@ -277,19 +276,6 @@ async def posts_reconcile_duplicates():
|
|||||||
return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
|
return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
|
||||||
|
|
||||||
|
|
||||||
@admin_bp.route("/tags/purge-legacy", methods=["POST"])
|
|
||||||
async def tags_purge_legacy():
|
|
||||||
"""Tier-A: delete legacy IR-migration tags — archive/post/artist
|
|
||||||
kinds (e.g. `BlenderKnight:Hannah_BJ_Loops`) PLUS general tags with
|
|
||||||
a legacy name prefix (`source:*`, from IR's source kind that fell
|
|
||||||
back to general). dry-run preview returns per-kind + per-prefix
|
|
||||||
counts + a sample so the UI shows exactly what'll go before the
|
|
||||||
operator confirms with dry_run=false."""
|
|
||||||
from ..services.cleanup_service import purge_legacy_tags
|
|
||||||
|
|
||||||
return await _run_dry_run_op(purge_legacy_tags)
|
|
||||||
|
|
||||||
|
|
||||||
@admin_bp.route("/tags/reset-content", methods=["POST"])
|
@admin_bp.route("/tags/reset-content", methods=["POST"])
|
||||||
async def tags_reset_content():
|
async def tags_reset_content():
|
||||||
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
|
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
|
||||||
|
|||||||
@@ -97,10 +97,6 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
"ml-backfill-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.backfill",
|
|
||||||
"schedule": 86400.0,
|
|
||||||
},
|
|
||||||
"train-heads-nightly": {
|
"train-heads-nightly": {
|
||||||
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
||||||
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
||||||
|
|||||||
@@ -718,71 +718,6 @@ def reconcile_duplicate_posts(
|
|||||||
return {"groups": len(groups), "merged": losers_total, "sample": sample}
|
return {"groups": len(groups), "merged": losers_total, "sample": sample}
|
||||||
|
|
||||||
|
|
||||||
# Legacy tags FC no longer uses, in two shapes:
|
|
||||||
# (1) kinds the tag input never produces — archive/post/artist.
|
|
||||||
# provenance (post grouping) + archive membership are their own
|
|
||||||
# systems now, and artists are first-class Artist/Source rows.
|
|
||||||
# meta/rating were already hard-deleted by alembic 0023.
|
|
||||||
# (2) name prefixes from IR kinds FC never adopted — `source:*`.
|
|
||||||
# ImageRepo had a `source` kind; FC's enum doesn't, so ir_ingest
|
|
||||||
# fell those back to `general` (kind=general, name="source:patreon"
|
|
||||||
# etc.). They can't be caught by kind, so we match the name prefix.
|
|
||||||
PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
|
|
||||||
LEGACY_NAME_PREFIXES = ("source:",)
|
|
||||||
|
|
||||||
|
|
||||||
def _legacy_tag_predicate():
|
|
||||||
name_clauses = [Tag.name.like(f"{p}%") for p in LEGACY_NAME_PREFIXES]
|
|
||||||
return or_(Tag.kind.in_(PURGEABLE_TAG_KINDS), *name_clauses)
|
|
||||||
|
|
||||||
|
|
||||||
def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
|
|
||||||
"""Count (dry_run) or delete legacy IR-migration tags: archive/post/
|
|
||||||
artist-kind tags PLUS general tags whose name matches a legacy
|
|
||||||
prefix (source:*).
|
|
||||||
|
|
||||||
CASCADE on image_tag / tag_alias / tag_suggestion_rejection / series_page
|
|
||||||
clears the related rows on the parent DELETE.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
{"by_kind": {kind: count, ...}, # kind-matched rows
|
|
||||||
"by_prefix": {"source:*": count}, # name-prefix-matched rows
|
|
||||||
"count": total, "sample_names": [first 50],
|
|
||||||
and on live runs "deleted": total}
|
|
||||||
"""
|
|
||||||
predicate = _legacy_tag_predicate()
|
|
||||||
rows = session.execute(
|
|
||||||
select(Tag.id, Tag.name, Tag.kind).where(predicate)
|
|
||||||
).all()
|
|
||||||
by_kind: dict[str, int] = {}
|
|
||||||
by_prefix: dict[str, int] = {}
|
|
||||||
for _id, name, kind in rows:
|
|
||||||
# Classify by name-prefix first so a source:* row counts once,
|
|
||||||
# under the prefix bucket, regardless of its (general) kind.
|
|
||||||
matched_prefix = next(
|
|
||||||
(p for p in LEGACY_NAME_PREFIXES if name.startswith(p)), None,
|
|
||||||
)
|
|
||||||
if matched_prefix is not None:
|
|
||||||
label = f"{matched_prefix}*"
|
|
||||||
by_prefix[label] = by_prefix.get(label, 0) + 1
|
|
||||||
else:
|
|
||||||
key = kind.value if hasattr(kind, "value") else str(kind)
|
|
||||||
by_kind[key] = by_kind.get(key, 0) + 1
|
|
||||||
sample = [name for _id, name, _kind in rows[:50]]
|
|
||||||
total = len(rows)
|
|
||||||
result = {
|
|
||||||
"by_kind": by_kind, "by_prefix": by_prefix,
|
|
||||||
"count": total, "sample_names": sample,
|
|
||||||
}
|
|
||||||
if dry_run:
|
|
||||||
return result
|
|
||||||
if total:
|
|
||||||
session.execute(Tag.__table__.delete().where(predicate))
|
|
||||||
session.commit()
|
|
||||||
result["deleted"] = total
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# The CONTENT vocabulary. "Reset content tagging" wipes these so the operator
|
# The CONTENT vocabulary. "Reset content tagging" wipes these so the operator
|
||||||
# can re-tag from scratch. fandom + series (and series_page ordering) are
|
# can re-tag from scratch. fandom + series (and series_page ordering) are
|
||||||
# deliberately NOT here — they're kept.
|
# deliberately NOT here — they're kept.
|
||||||
|
|||||||
@@ -42,53 +42,6 @@
|
|||||||
</div>
|
</div>
|
||||||
</MaintenanceTile>
|
</MaintenanceTile>
|
||||||
|
|
||||||
<MaintenanceTile
|
|
||||||
icon="mdi-tag-off"
|
|
||||||
title="Legacy migration tags"
|
|
||||||
blurb="Purge retired archive/post/artist + source:* tags."
|
|
||||||
destructive
|
|
||||||
>
|
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
|
||||||
Purge legacy IR-migration tags FC no longer uses: retired/system
|
|
||||||
kinds (<code>archive</code>, <code>post</code>, <code>artist</code> — e.g.
|
|
||||||
<code>BlenderKnight:Hannah_BJ_Loops</code>) plus <code>source:*</code> tags
|
|
||||||
(ImageRepo's old <code>source</code> kind, migrated to <code>general</code>).
|
|
||||||
Provenance and artists are their own systems now, so these are pure noise.
|
|
||||||
Removes them from every image.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<v-btn
|
|
||||||
color="accent" variant="flat" rounded="pill"
|
|
||||||
prepend-icon="mdi-magnify"
|
|
||||||
:loading="loadingKindPreview"
|
|
||||||
class="mb-3"
|
|
||||||
@click="onKindPreview"
|
|
||||||
>Preview legacy tags</v-btn>
|
|
||||||
|
|
||||||
<div v-if="kindPreview">
|
|
||||||
<p class="text-body-2 mb-2">
|
|
||||||
<strong>{{ kindPreview.count }}</strong> legacy tag(s).
|
|
||||||
<span v-for="(n, k) in kindPreview.by_kind" :key="k" class="fc-muted">
|
|
||||||
{{ k }}: {{ n }}
|
|
||||||
</span>
|
|
||||||
<span v-for="(n, p) in kindPreview.by_prefix" :key="p" class="fc-muted">
|
|
||||||
{{ p }}: {{ n }}
|
|
||||||
</span>
|
|
||||||
</p>
|
|
||||||
<SampleNameGrid
|
|
||||||
v-if="kindPreview.sample_names?.length"
|
|
||||||
:names="kindPreview.sample_names" class="mb-3"
|
|
||||||
/>
|
|
||||||
<v-btn
|
|
||||||
color="error" variant="flat" rounded="pill"
|
|
||||||
prepend-icon="mdi-delete-sweep"
|
|
||||||
:disabled="!kindPreview.count"
|
|
||||||
:loading="kindCommitting"
|
|
||||||
@click="onKindCommit"
|
|
||||||
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
|
|
||||||
</div>
|
|
||||||
</MaintenanceTile>
|
|
||||||
|
|
||||||
<MaintenanceTile
|
<MaintenanceTile
|
||||||
icon="mdi-tag-multiple"
|
icon="mdi-tag-multiple"
|
||||||
title="Reset content tagging"
|
title="Reset content tagging"
|
||||||
@@ -216,16 +169,6 @@ const {
|
|||||||
emptyPreview: (r) => ({ count: 0, sample_names: r.sample_names || [] }),
|
emptyPreview: (r) => ({ count: 0, sample_names: r.sample_names || [] }),
|
||||||
})
|
})
|
||||||
|
|
||||||
// Legacy migration-tag purge.
|
|
||||||
const {
|
|
||||||
previewData: kindPreview, previewing: loadingKindPreview,
|
|
||||||
committing: kindCommitting, runPreview: onKindPreview, runCommit: onKindCommit,
|
|
||||||
} = usePreviewCommit({
|
|
||||||
preview: () => store.purgeLegacyTags({ dryRun: true }),
|
|
||||||
commit: () => store.purgeLegacyTags({ dryRun: false }),
|
|
||||||
emptyPreview: { count: 0, by_kind: {}, by_prefix: {}, sample_names: [] },
|
|
||||||
})
|
|
||||||
|
|
||||||
// Reset content tagging (general + character).
|
// Reset content tagging (general + character).
|
||||||
const {
|
const {
|
||||||
previewData: resetPreview, previewing: loadingResetPreview,
|
previewData: resetPreview, previewing: loadingResetPreview,
|
||||||
|
|||||||
@@ -101,10 +101,6 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function purgeLegacyTags(opts = {}) {
|
|
||||||
return _dryRunPost('/api/admin/tags/purge-legacy', opts)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Destructive: deletes ALL general + character tags so the operator can
|
// Destructive: deletes ALL general + character tags so the operator can
|
||||||
// re-tag from scratch via auto-suggest. fandom + series preserved.
|
// re-tag from scratch via auto-suggest. fandom + series preserved.
|
||||||
function resetContentTagging(opts = {}) {
|
function resetContentTagging(opts = {}) {
|
||||||
@@ -154,7 +150,6 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
pruneUnusedTags,
|
pruneUnusedTags,
|
||||||
pruneBarePosts,
|
pruneBarePosts,
|
||||||
reconcileDuplicatePosts,
|
reconcileDuplicatePosts,
|
||||||
purgeLegacyTags,
|
|
||||||
resetContentTagging,
|
resetContentTagging,
|
||||||
normalizeTags,
|
normalizeTags,
|
||||||
pollTaskUntilDone,
|
pollTaskUntilDone,
|
||||||
|
|||||||
@@ -390,63 +390,6 @@ async def test_prune_unused_commit_deletes_and_returns_count(client, db):
|
|||||||
assert "byebye" in body["sample_names"]
|
assert "byebye" in body["sample_names"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_purge_legacy_dry_run_counts_by_kind_and_prefix(client, db):
|
|
||||||
db.add_all([
|
|
||||||
Tag(name="BlenderKnight:Hannah_BJ_Loops", kind=TagKind.archive),
|
|
||||||
Tag(name="BlenderKnight:May Animation", kind=TagKind.post),
|
|
||||||
Tag(name="SomeArtist", kind=TagKind.artist),
|
|
||||||
# IR `source` kind fell back to general during migration —
|
|
||||||
# caught by the source:* name prefix, not by kind.
|
|
||||||
Tag(name="source:patreon", kind=TagKind.general),
|
|
||||||
Tag(name="blonde hair", kind=TagKind.general), # must survive
|
|
||||||
])
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
resp = await client.post(
|
|
||||||
"/api/admin/tags/purge-legacy", json={"dry_run": True},
|
|
||||||
)
|
|
||||||
body = await resp.get_json()
|
|
||||||
assert body["count"] == 4
|
|
||||||
assert body["by_kind"] == {"archive": 1, "post": 1, "artist": 1}
|
|
||||||
assert body["by_prefix"] == {"source:*": 1}
|
|
||||||
assert "blonde hair" not in body["sample_names"]
|
|
||||||
assert "source:patreon" in body["sample_names"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_purge_legacy_commit_deletes_only_legacy(client, db):
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
|
|
||||||
db.add_all([
|
|
||||||
Tag(name="arch1", kind=TagKind.archive),
|
|
||||||
Tag(name="source:fanbox", kind=TagKind.general),
|
|
||||||
Tag(name="keepme", kind=TagKind.general),
|
|
||||||
Tag(name="char1", kind=TagKind.character),
|
|
||||||
])
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
resp = await client.post(
|
|
||||||
"/api/admin/tags/purge-legacy", json={"dry_run": False},
|
|
||||||
)
|
|
||||||
body = await resp.get_json()
|
|
||||||
assert body["deleted"] == 2 # arch1 + source:fanbox
|
|
||||||
|
|
||||||
# The plain general + character tags survive.
|
|
||||||
keep = (await db.execute(
|
|
||||||
select(Tag.name).where(Tag.name.in_(["keepme", "char1"]))
|
|
||||||
)).scalars().all()
|
|
||||||
assert set(keep) == {"keepme", "char1"}
|
|
||||||
# No archive/post/artist or source:* left.
|
|
||||||
gone = (await db.execute(
|
|
||||||
select(func.count()).select_from(Tag).where(
|
|
||||||
(Tag.kind.in_([TagKind.archive, TagKind.post, TagKind.artist]))
|
|
||||||
| (Tag.name.like("source:%"))
|
|
||||||
)
|
|
||||||
)).scalar_one()
|
|
||||||
assert gone == 0
|
|
||||||
|
|
||||||
|
|
||||||
# --- Tier-A: POST /posts/prune-bare + /posts/reconcile-duplicates ---
|
# --- Tier-A: POST /posts/prune-bare + /posts/reconcile-duplicates ---
|
||||||
# These two routes share _run_dry_run_op with the tag prunes (DRY pass,
|
# These two routes share _run_dry_run_op with the tag prunes (DRY pass,
|
||||||
# task #753); cover their apply path + reconcile's source_id passthrough so
|
# task #753); cover their apply path + reconcile's source_id passthrough so
|
||||||
|
|||||||
Reference in New Issue
Block a user