dev→main: similar-search render fix + reset-content-tagging + scan persistence #66

Merged
bvandeusen merged 3 commits from dev into main 2026-06-04 16:59:52 -04:00
13 changed files with 363 additions and 6 deletions
+21
View File
@@ -224,6 +224,27 @@ async def tags_purge_legacy():
return jsonify(result)
@admin_bp.route("/tags/reset-content", methods=["POST"])
async def tags_reset_content():
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
content vocabulary) so the operator can re-tag from scratch via
auto-suggest. fandom + series tags + series_page ordering are preserved,
and image tagger_predictions are untouched so suggestions repopulate.
dry-run preview returns per-kind counts + applications + a sample so the
UI shows exactly what'll go before the operator confirms (dry_run=false).
Irreversible except via DB backup restore."""
from ..services.cleanup_service import reset_content_tagging
body = await request.get_json(silent=True) or {}
dry_run = bool(body.get("dry_run", False))
async with get_session() as session:
result = await session.run_sync(
lambda sync_sess: reset_content_tagging(sync_sess, dry_run=dry_run)
)
return jsonify(result)
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
async def db_stats():
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
+8 -5
View File
@@ -154,12 +154,15 @@ async def audit_history():
limit = min(int(request.args.get("limit", "20")), 100)
except ValueError:
return _bad("invalid_limit")
# Optional rule filter so a card can reconnect to ITS latest run on mount
# (?rule=transparency&limit=1) — the audit survives navigation; the UI
# rehydrates from this rather than losing the in-flight scan.
rule = request.args.get("rule") or None
async with get_session() as session:
rows = (await session.execute(
select(LibraryAuditRun)
.order_by(LibraryAuditRun.id.desc())
.limit(limit)
)).scalars().all()
stmt = select(LibraryAuditRun).order_by(LibraryAuditRun.id.desc())
if rule is not None:
stmt = stmt.where(LibraryAuditRun.rule == rule)
rows = (await session.execute(stmt.limit(limit))).scalars().all()
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
+56
View File
@@ -455,6 +455,62 @@ def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
return result
# The Camie-suggestable CONTENT vocabulary. "Reset content tagging" wipes
# these so the operator can re-tag from scratch via auto-suggest. fandom +
# series (and series_page ordering) are deliberately NOT here — they're kept.
RESETTABLE_TAG_KINDS = ("general", "character")
def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
"""Count (dry_run) or DELETE every general + character tag so the operator
can re-tag from scratch via the Camie auto-suggest.
PRESERVED: fandom + series tags and their series_page ordering, plus every
image's image_record.tagger_predictions (untouched) so suggestions
repopulate immediately. CASCADE on image_tag / tag_alias / tag_allowlist /
tag_reference_embedding / tag_suggestion_rejection clears each deleted
tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting
character tags never touches the fandom rows. Irreversible except via DB
backup restore.
Returns:
{"by_kind": {"general": N, "character": M},
"count": total tags,
"applications": image_tag rows that will be / were removed,
"sample_names": [first 50],
and on live runs "deleted": total}
"""
predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
rows = session.execute(
select(Tag.id, Tag.name, Tag.kind).where(predicate)
).all()
by_kind: dict[str, int] = {}
for _id, _name, kind in rows:
key = kind.value if hasattr(kind, "value") else str(kind)
by_kind[key] = by_kind.get(key, 0) + 1
# Headline impact: applications (image_tag rows) that vanish via cascade.
applications = session.execute(
select(func.count())
.select_from(image_tag)
.where(image_tag.c.tag_id.in_(select(Tag.id).where(predicate)))
).scalar_one()
sample = [name for _id, name, _kind in rows[:50]]
total = len(rows)
result = {
"by_kind": by_kind,
"count": total,
"applications": applications,
"sample_names": sample,
}
if dry_run:
return result
if total:
session.execute(Tag.__table__.delete().where(predicate))
session.commit()
result["deleted"] = total
return result
# ---------------------------------------------------------------------------
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
# ---------------------------------------------------------------------------
@@ -112,6 +112,16 @@ onMounted(async () => {
await store.loadDefaults()
threshold.value = store.defaults.single_color_threshold
tolerance.value = store.defaults.single_color_tolerance
// Reconnect to this rule's latest run so a scan started before navigating
// away keeps showing progress / its result on return (the scan itself runs
// backend-side regardless).
try {
const latest = await store.latestAuditForRule('single_color')
if (latest) {
audit.value = latest
if (latest.status === 'running') startPoll(latest.id)
}
} catch { /* non-fatal — card still works for a fresh scan */ }
})
onUnmounted(() => stopPoll())
@@ -97,6 +97,16 @@ let pollTimer = null
onMounted(async () => {
await store.loadDefaults()
threshold.value = store.defaults.transparency_threshold
// Reconnect to this rule's latest run so a scan started before navigating
// away keeps showing progress / its result on return (the scan itself runs
// backend-side regardless).
try {
const latest = await store.latestAuditForRule('transparency')
if (latest) {
audit.value = latest
if (latest.status === 'running') startPoll(latest.id)
}
} catch { /* non-fatal — card still works for a fresh scan */ }
})
onUnmounted(() => stopPoll())
@@ -15,6 +15,19 @@
</div>
</template>
<!-- Similar-mode (and any non-date-grouped result set) returns no date
groups the results are ranked, not chronological. Render them as a
single flat list in their given order rather than nothing. -->
<div
v-if="!store.dateGroups.length && store.images.length"
class="fc-gallery-grid__items"
>
<GalleryItem
v-for="img in store.images"
:key="img.id" :image="img" @open="$emit('open', img.id)"
/>
</div>
<div v-if="store.loading" class="fc-gallery-grid__sentinel">
<v-progress-circular indeterminate color="accent" size="28" />
</div>
@@ -89,6 +89,53 @@
@click="onKindCommit"
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
</div>
<v-divider class="my-5" />
<p class="text-body-2 mb-2">
<strong class="text-error">Reset content tagging.</strong>
Deletes every <code>general</code> and <code>character</code> tag and
removes them from every image, so you can re-tag from scratch with the
auto-suggest. <strong>Fandoms and series (with their page order) are
kept</strong>, and each image's saved predictions are untouched open
an image and its suggestions reappear.
</p>
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
Irreversible there's no undo except restoring a DB backup.
Back one up first (Settings → Maintenance → Backup).
</v-alert>
<v-btn
color="accent" variant="flat" rounded="pill"
prepend-icon="mdi-magnify"
:loading="loadingResetPreview"
class="mb-3"
@click="onResetPreview"
>Preview content-tag reset</v-btn>
<div v-if="resetPreview">
<p class="text-body-2 mb-2">
<strong>{{ resetPreview.count }}</strong> content tag(s)
<span v-for="(n, k) in resetPreview.by_kind" :key="k" class="fc-muted">
({{ k }}: {{ n }})&nbsp;
</span>
across <strong>{{ resetPreview.applications }}</strong> image
application(s).
</p>
<div v-if="resetPreview.sample_names?.length" class="fc-name-grid mb-3">
<span v-for="n in resetPreview.sample_names" :key="n" class="fc-name">
{{ n }}
</span>
</div>
<v-btn
color="error" variant="flat" rounded="pill"
prepend-icon="mdi-delete-alert"
:disabled="!resetPreview.count"
:loading="resetCommitting"
@click="onResetCommit"
>Delete {{ resetPreview.count }} content tag(s) +
{{ resetPreview.applications }} application(s)</v-btn>
</div>
</v-card-text>
</v-card>
</template>
@@ -105,6 +152,9 @@ const committing = ref(false)
const kindPreview = ref(null)
const loadingKindPreview = ref(false)
const kindCommitting = ref(false)
const resetPreview = ref(null)
const loadingResetPreview = ref(false)
const resetCommitting = ref(false)
async function onPreview() {
loadingPreview.value = true
@@ -143,6 +193,25 @@ async function onKindCommit() {
kindCommitting.value = false
}
}
async function onResetPreview() {
loadingResetPreview.value = true
try {
resetPreview.value = await store.resetContentTagging({ dryRun: true })
} finally {
loadingResetPreview.value = false
}
}
async function onResetCommit() {
resetCommitting.value = true
try {
await store.resetContentTagging({ dryRun: false })
resetPreview.value = { count: 0, by_kind: {}, applications: 0, sample_names: [] }
} finally {
resetCommitting.value = false
}
}
</script>
<style scoped>
+16
View File
@@ -127,6 +127,21 @@ export const useAdminStore = defineStore('admin', () => {
}
}
// Destructive: deletes ALL general + character tags so the operator can
// re-tag from scratch via auto-suggest. fandom + series preserved.
async function resetContentTagging({ dryRun = true } = {}) {
lastError.value = null
try {
return await api.post(
'/api/admin/tags/reset-content',
{ body: { dry_run: dryRun } },
)
} catch (e) {
lastError.value = e.message
throw e
}
}
// --- Task progress polling (taps FC-3i activity dashboard) --------
/**
@@ -162,6 +177,7 @@ export const useAdminStore = defineStore('admin', () => {
tagUsageCount,
pruneUnusedTags,
purgeLegacyTags,
resetContentTagging,
pollTaskUntilDone,
}
})
+9 -1
View File
@@ -55,6 +55,14 @@ export const useCleanupStore = defineStore('cleanup', () => {
return body.runs
}
// The most recent audit run for a given rule, or null. Cards call this on
// mount to reconnect to a scan that's still running (or to show the last
// completed result) after the user navigates away and back.
async function latestAuditForRule(rule) {
const body = await api.get('/api/cleanup/audit', { params: { rule, limit: 1 } })
return (body.runs && body.runs[0]) || null
}
async function applyAudit(id, confirm) {
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
}
@@ -67,6 +75,6 @@ export const useCleanupStore = defineStore('cleanup', () => {
defaults, recentRuns,
loadDefaults,
previewMinDim, deleteMinDim,
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
startAudit, getAudit, loadHistory, latestAuditForRule, applyAudit, cancelAudit,
}
})
@@ -0,0 +1,47 @@
// @vitest-environment happy-dom
import { describe, it, expect } from 'vitest'
import { mount } from '@vue/test-utils'
import GalleryGrid from '../../src/components/gallery/GalleryGrid.vue'
import { useGalleryStore } from '../../src/stores/gallery.js'
import { freshPinia } from '../support/mountComponent.js'
const GIStub = { name: 'GalleryItem', props: ['image'], template: '<div class="gi" />' }
function mountGrid(pinia) {
return mount(GalleryGrid, {
global: {
plugins: [pinia],
stubs: { GalleryItem: GIStub, RouterLink: { template: '<a><slot /></a>' } },
},
})
}
describe('GalleryGrid', () => {
it('renders a flat ranked list when there are no date groups (similar mode)', () => {
const pinia = freshPinia()
const store = useGalleryStore()
// similar-mode shape: images present, date_groups empty, no cursor.
store.images = [
{ id: 1, thumbnail_url: '/a' },
{ id: 2, thumbnail_url: '/b' },
{ id: 3, thumbnail_url: '/c' },
]
store.dateGroups = []
const w = mountGrid(pinia)
expect(w.findAll('.gi').length).toBe(3)
})
it('renders grouped by date when date groups are present', () => {
const pinia = freshPinia()
const store = useGalleryStore()
store.images = [
{ id: 1, thumbnail_url: '/a' },
{ id: 2, thumbnail_url: '/b' },
]
store.dateGroups = [{ year: 2026, month: 6, image_ids: [1, 2] }]
const w = mountGrid(pinia)
expect(w.find('.fc-gallery-grid__date-header').exists()).toBe(true)
expect(w.findAll('.gi').length).toBe(2)
})
})
+23
View File
@@ -435,3 +435,26 @@ async def test_trigger_vacuum_queues_the_task(client, monkeypatch):
resp = await client.post("/api/admin/maintenance/vacuum")
assert resp.status_code == 202
assert calls == [1]
# --- Tier-A: POST /tags/reset-content -------------------------------
@pytest.mark.asyncio
async def test_reset_content_tagging_dry_run_returns_counts(client, db):
db.add_all([
Tag(name="solo", kind=TagKind.general),
Tag(name="naruto", kind=TagKind.character),
Tag(name="Naruto", kind=TagKind.fandom),
Tag(name="my-series", kind=TagKind.series),
])
await db.commit()
resp = await client.post(
"/api/admin/tags/reset-content", json={"dry_run": True}
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["count"] == 2
assert body["by_kind"] == {"general": 1, "character": 1}
# dry-run leaves the rows in place — fandom + series untouched too.
assert "deleted" not in body
+21
View File
@@ -155,6 +155,27 @@ async def test_audit_history_returns_recent_runs(client, db):
assert len(body["runs"]) >= 3
@pytest.mark.asyncio
async def test_audit_history_filters_by_rule(client, db):
db.add(LibraryAuditRun(
rule="transparency", params={"threshold": 0.9},
status="applied", matched_ids=[], finished_at=datetime.now(UTC),
))
db.add(LibraryAuditRun(
rule="single_color", params={"threshold": 0.95, "tolerance": 30},
status="ready", matched_count=2, matched_ids=[1, 2],
finished_at=datetime.now(UTC),
))
await db.commit()
# ?rule=&limit=1 → just THIS rule's latest run (the card-reconnect query).
resp = await client.get("/api/cleanup/audit?rule=single_color&limit=1")
assert resp.status_code == 200
runs = (await resp.get_json())["runs"]
assert len(runs) == 1
assert runs[0]["rule"] == "single_color"
assert runs[0]["matched_count"] == 2
@pytest.mark.asyncio
async def test_audit_apply_with_token_deletes(client, db, tmp_path):
rec = await _seed_image(db, tmp_path, w=100, h=100, name="apply.png")
+60
View File
@@ -9,6 +9,7 @@ import pytest
from sqlalchemy import func, select
from backend.app.models import Artist, ImageRecord, Tag, TagKind
from backend.app.models.series_page import SeriesPage
from backend.app.models.tag import image_tag
from backend.app.services import cleanup_service
@@ -333,3 +334,62 @@ def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
assert "kept" in surviving_names
assert "bye" not in surviving_names
# --- reset_content_tagging ------------------------------------------
def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path):
a = _make_artist(db_sync, slug="rc")
img = _make_image(
db_sync, artist=a, path=str(tmp_path / "r.jpg"), sha256="1" * 64,
)
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
_make_tag(db_sync, name="my-series", kind=TagKind.series)
db_sync.execute(image_tag.insert().values([
{"image_record_id": img.id, "tag_id": g.id},
{"image_record_id": img.id, "tag_id": c.id},
]))
db_sync.commit()
result = cleanup_service.reset_content_tagging(db_sync, dry_run=True)
assert result["count"] == 2
assert result["by_kind"] == {"general": 1, "character": 1}
assert result["applications"] == 2
# Nothing deleted — all 4 tags still present.
assert db_sync.execute(select(func.count(Tag.id))).scalar_one() == 4
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
a = _make_artist(db_sync, slug="rc2")
img = _make_image(
db_sync, artist=a, path=str(tmp_path / "r2.jpg"), sha256="2" * 64,
)
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
s = _make_tag(db_sync, name="my-series", kind=TagKind.series)
db_sync.execute(image_tag.insert().values([
{"image_record_id": img.id, "tag_id": g.id},
{"image_record_id": img.id, "tag_id": c.id},
{"image_record_id": img.id, "tag_id": s.id}, # series membership
]))
db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1))
db_sync.commit()
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
assert result["deleted"] == 2
# general + character gone; fandom + series kept.
kinds = db_sync.execute(select(Tag.kind)).scalars().all()
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
assert kind_vals == {"fandom", "series"}
# series_page ordering survived.
assert db_sync.execute(
select(func.count()).select_from(SeriesPage)
).scalar_one() == 1
# Only the series image_tag association survived (content ones cascaded).
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
assert remaining == [s.id]