Compare commits

...
4 Commits
Author SHA1 Message Date
bvandeusen 81688815a0 Merge pull request 'dev→main: similar-search render fix + reset-content-tagging + scan persistence' (#66) from dev into main
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 2s
Build images / build-ml (push) Successful in 9s
CI / backend-lint-and-test (push) Successful in 13s
Build images / build-web (push) Successful in 11s
CI / frontend-build (push) Successful in 22s
CI / integration (push) Successful in 2m58s
2026-06-04 16:59:52 -04:00
bvandeusenandClaude Opus 4.8 5a6a95682d fix(cleanup): library scans survive navigation, reconnect on return
CI / frontend-build (push) Successful in 24s
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Successful in 11s
CI / integration (push) Successful in 2m56s
The transparency / single-color audit cards held the run + poll timer in
local component state, so navigating away destroyed both and onMounted never
reconnected — the Celery scan kept running and writing LibraryAuditRun, but
the UI forgot it. Now each card, on mount, fetches its rule's latest run
(GET /api/cleanup/audit?rule=<rule>&limit=1) and rehydrates: shows progress +
resumes polling if still running, or shows the completed result (ready/applied/
error) so the operator can act on it after returning. Adds the ?rule= filter
to the audit-history endpoint + cleanup store latestAuditForRule().

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-04 16:55:20 -04:00
bvandeusenandClaude Opus 4.8 91b0145bc8 feat(tags): 'Reset content tagging' admin action
CI / backend-lint-and-test (push) Successful in 11s
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 30s
CI / integration (push) Successful in 2m57s
Wipe every general + character tag so the operator can re-tag from scratch via
the Camie auto-suggest, while PRESERVING fandoms, series (+ series_page order),
and each image's stored tagger_predictions (so suggestions repopulate
immediately). One set-based DELETE FROM tag WHERE kind IN ('general','character')
— the five tag-referencing tables all cascade, so applications + aliases +
allowlist + rejections + centroids clear automatically; series tags aren't
deleted so series survive; Tag.fandom_id is SET NULL so fandoms are untouched.

Reuses the established dry-run-preview -> confirm pattern: cleanup_service.
reset_content_tagging() + POST /api/admin/tags/reset-content +
TagMaintenanceCard section with a backup-first warning and a red confirm
showing exact counts (tags by kind + image applications). Irreversible except
via DB backup restore; the wipe only fires when the operator confirms.

Tests: service dry-run counts + live delete preserves fandom/series/series_page
while content tags + their image_tag cascade away; API dry-run wiring.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-04 16:47:36 -04:00
bvandeusenandClaude Opus 4.8 26e47a86cb fix(gallery): render similar-mode results (flat list when no date groups)
CI / lint (push) Successful in 3s
CI / backend-lint-and-test (push) Successful in 12s
CI / frontend-build (push) Successful in 22s
CI / integration (push) Successful in 2m58s
The 'See all similar' takeover fetched /api/gallery/similar fine (200, ~100
results) but the grid showed nothing: GalleryGrid renders ONLY by iterating
store.dateGroups, and similar-mode returns date_groups=[] (results are ranked
by cosine distance, not chronological). Zero groups → zero tiles despite
store.images being full. Add a flat fallback: when there are no date groups
but images exist, render them as one ungrouped list in ranked order (no date
headers). The modal Related strip was unaffected (it renders its images
directly). Test locks both the flat and grouped paths.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-04 16:24:52 -04:00
13 changed files with 363 additions and 6 deletions
+21
View File
@@ -224,6 +224,27 @@ async def tags_purge_legacy():
return jsonify(result)
@admin_bp.route("/tags/reset-content", methods=["POST"])
async def tags_reset_content():
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
content vocabulary) so the operator can re-tag from scratch via
auto-suggest. fandom + series tags + series_page ordering are preserved,
and image tagger_predictions are untouched so suggestions repopulate.
dry-run preview returns per-kind counts + applications + a sample so the
UI shows exactly what'll go before the operator confirms (dry_run=false).
Irreversible except via DB backup restore."""
from ..services.cleanup_service import reset_content_tagging
body = await request.get_json(silent=True) or {}
dry_run = bool(body.get("dry_run", False))
async with get_session() as session:
result = await session.run_sync(
lambda sync_sess: reset_content_tagging(sync_sess, dry_run=dry_run)
)
return jsonify(result)
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
async def db_stats():
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
+8 -5
View File
@@ -154,12 +154,15 @@ async def audit_history():
limit = min(int(request.args.get("limit", "20")), 100)
except ValueError:
return _bad("invalid_limit")
# Optional rule filter so a card can reconnect to ITS latest run on mount
# (?rule=transparency&limit=1) — the audit survives navigation; the UI
# rehydrates from this rather than losing the in-flight scan.
rule = request.args.get("rule") or None
async with get_session() as session:
rows = (await session.execute(
select(LibraryAuditRun)
.order_by(LibraryAuditRun.id.desc())
.limit(limit)
)).scalars().all()
stmt = select(LibraryAuditRun).order_by(LibraryAuditRun.id.desc())
if rule is not None:
stmt = stmt.where(LibraryAuditRun.rule == rule)
rows = (await session.execute(stmt.limit(limit))).scalars().all()
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
+56
View File
@@ -455,6 +455,62 @@ def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
return result
# The Camie-suggestable CONTENT vocabulary. "Reset content tagging" wipes
# these so the operator can re-tag from scratch via auto-suggest. fandom +
# series (and series_page ordering) are deliberately NOT here — they're kept.
RESETTABLE_TAG_KINDS = ("general", "character")
def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
"""Count (dry_run) or DELETE every general + character tag so the operator
can re-tag from scratch via the Camie auto-suggest.
PRESERVED: fandom + series tags and their series_page ordering, plus every
image's image_record.tagger_predictions (untouched) so suggestions
repopulate immediately. CASCADE on image_tag / tag_alias / tag_allowlist /
tag_reference_embedding / tag_suggestion_rejection clears each deleted
tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting
character tags never touches the fandom rows. Irreversible except via DB
backup restore.
Returns:
{"by_kind": {"general": N, "character": M},
"count": total tags,
"applications": image_tag rows that will be / were removed,
"sample_names": [first 50],
and on live runs "deleted": total}
"""
predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
rows = session.execute(
select(Tag.id, Tag.name, Tag.kind).where(predicate)
).all()
by_kind: dict[str, int] = {}
for _id, _name, kind in rows:
key = kind.value if hasattr(kind, "value") else str(kind)
by_kind[key] = by_kind.get(key, 0) + 1
# Headline impact: applications (image_tag rows) that vanish via cascade.
applications = session.execute(
select(func.count())
.select_from(image_tag)
.where(image_tag.c.tag_id.in_(select(Tag.id).where(predicate)))
).scalar_one()
sample = [name for _id, name, _kind in rows[:50]]
total = len(rows)
result = {
"by_kind": by_kind,
"count": total,
"applications": applications,
"sample_names": sample,
}
if dry_run:
return result
if total:
session.execute(Tag.__table__.delete().where(predicate))
session.commit()
result["deleted"] = total
return result
# ---------------------------------------------------------------------------
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
# ---------------------------------------------------------------------------
@@ -112,6 +112,16 @@ onMounted(async () => {
await store.loadDefaults()
threshold.value = store.defaults.single_color_threshold
tolerance.value = store.defaults.single_color_tolerance
// Reconnect to this rule's latest run so a scan started before navigating
// away keeps showing progress / its result on return (the scan itself runs
// backend-side regardless).
try {
const latest = await store.latestAuditForRule('single_color')
if (latest) {
audit.value = latest
if (latest.status === 'running') startPoll(latest.id)
}
} catch { /* non-fatal — card still works for a fresh scan */ }
})
onUnmounted(() => stopPoll())
@@ -97,6 +97,16 @@ let pollTimer = null
onMounted(async () => {
await store.loadDefaults()
threshold.value = store.defaults.transparency_threshold
// Reconnect to this rule's latest run so a scan started before navigating
// away keeps showing progress / its result on return (the scan itself runs
// backend-side regardless).
try {
const latest = await store.latestAuditForRule('transparency')
if (latest) {
audit.value = latest
if (latest.status === 'running') startPoll(latest.id)
}
} catch { /* non-fatal — card still works for a fresh scan */ }
})
onUnmounted(() => stopPoll())
@@ -15,6 +15,19 @@
</div>
</template>
<!-- Similar-mode (and any non-date-grouped result set) returns no date
groups the results are ranked, not chronological. Render them as a
single flat list in their given order rather than nothing. -->
<div
v-if="!store.dateGroups.length && store.images.length"
class="fc-gallery-grid__items"
>
<GalleryItem
v-for="img in store.images"
:key="img.id" :image="img" @open="$emit('open', img.id)"
/>
</div>
<div v-if="store.loading" class="fc-gallery-grid__sentinel">
<v-progress-circular indeterminate color="accent" size="28" />
</div>
@@ -89,6 +89,53 @@
@click="onKindCommit"
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
</div>
<v-divider class="my-5" />
<p class="text-body-2 mb-2">
<strong class="text-error">Reset content tagging.</strong>
Deletes every <code>general</code> and <code>character</code> tag and
removes them from every image, so you can re-tag from scratch with the
auto-suggest. <strong>Fandoms and series (with their page order) are
kept</strong>, and each image's saved predictions are untouched open
an image and its suggestions reappear.
</p>
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
Irreversible there's no undo except restoring a DB backup.
Back one up first (Settings → Maintenance → Backup).
</v-alert>
<v-btn
color="accent" variant="flat" rounded="pill"
prepend-icon="mdi-magnify"
:loading="loadingResetPreview"
class="mb-3"
@click="onResetPreview"
>Preview content-tag reset</v-btn>
<div v-if="resetPreview">
<p class="text-body-2 mb-2">
<strong>{{ resetPreview.count }}</strong> content tag(s)
<span v-for="(n, k) in resetPreview.by_kind" :key="k" class="fc-muted">
({{ k }}: {{ n }})&nbsp;
</span>
across <strong>{{ resetPreview.applications }}</strong> image
application(s).
</p>
<div v-if="resetPreview.sample_names?.length" class="fc-name-grid mb-3">
<span v-for="n in resetPreview.sample_names" :key="n" class="fc-name">
{{ n }}
</span>
</div>
<v-btn
color="error" variant="flat" rounded="pill"
prepend-icon="mdi-delete-alert"
:disabled="!resetPreview.count"
:loading="resetCommitting"
@click="onResetCommit"
>Delete {{ resetPreview.count }} content tag(s) +
{{ resetPreview.applications }} application(s)</v-btn>
</div>
</v-card-text>
</v-card>
</template>
@@ -105,6 +152,9 @@ const committing = ref(false)
const kindPreview = ref(null)
const loadingKindPreview = ref(false)
const kindCommitting = ref(false)
const resetPreview = ref(null)
const loadingResetPreview = ref(false)
const resetCommitting = ref(false)
async function onPreview() {
loadingPreview.value = true
@@ -143,6 +193,25 @@ async function onKindCommit() {
kindCommitting.value = false
}
}
async function onResetPreview() {
loadingResetPreview.value = true
try {
resetPreview.value = await store.resetContentTagging({ dryRun: true })
} finally {
loadingResetPreview.value = false
}
}
async function onResetCommit() {
resetCommitting.value = true
try {
await store.resetContentTagging({ dryRun: false })
resetPreview.value = { count: 0, by_kind: {}, applications: 0, sample_names: [] }
} finally {
resetCommitting.value = false
}
}
</script>
<style scoped>
+16
View File
@@ -127,6 +127,21 @@ export const useAdminStore = defineStore('admin', () => {
}
}
// Destructive: deletes ALL general + character tags so the operator can
// re-tag from scratch via auto-suggest. fandom + series preserved.
async function resetContentTagging({ dryRun = true } = {}) {
lastError.value = null
try {
return await api.post(
'/api/admin/tags/reset-content',
{ body: { dry_run: dryRun } },
)
} catch (e) {
lastError.value = e.message
throw e
}
}
// --- Task progress polling (taps FC-3i activity dashboard) --------
/**
@@ -162,6 +177,7 @@ export const useAdminStore = defineStore('admin', () => {
tagUsageCount,
pruneUnusedTags,
purgeLegacyTags,
resetContentTagging,
pollTaskUntilDone,
}
})
+9 -1
View File
@@ -55,6 +55,14 @@ export const useCleanupStore = defineStore('cleanup', () => {
return body.runs
}
// The most recent audit run for a given rule, or null. Cards call this on
// mount to reconnect to a scan that's still running (or to show the last
// completed result) after the user navigates away and back.
async function latestAuditForRule(rule) {
const body = await api.get('/api/cleanup/audit', { params: { rule, limit: 1 } })
return (body.runs && body.runs[0]) || null
}
async function applyAudit(id, confirm) {
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
}
@@ -67,6 +75,6 @@ export const useCleanupStore = defineStore('cleanup', () => {
defaults, recentRuns,
loadDefaults,
previewMinDim, deleteMinDim,
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
startAudit, getAudit, loadHistory, latestAuditForRule, applyAudit, cancelAudit,
}
})
@@ -0,0 +1,47 @@
// @vitest-environment happy-dom
import { describe, it, expect } from 'vitest'
import { mount } from '@vue/test-utils'
import GalleryGrid from '../../src/components/gallery/GalleryGrid.vue'
import { useGalleryStore } from '../../src/stores/gallery.js'
import { freshPinia } from '../support/mountComponent.js'
const GIStub = { name: 'GalleryItem', props: ['image'], template: '<div class="gi" />' }
function mountGrid(pinia) {
return mount(GalleryGrid, {
global: {
plugins: [pinia],
stubs: { GalleryItem: GIStub, RouterLink: { template: '<a><slot /></a>' } },
},
})
}
describe('GalleryGrid', () => {
it('renders a flat ranked list when there are no date groups (similar mode)', () => {
const pinia = freshPinia()
const store = useGalleryStore()
// similar-mode shape: images present, date_groups empty, no cursor.
store.images = [
{ id: 1, thumbnail_url: '/a' },
{ id: 2, thumbnail_url: '/b' },
{ id: 3, thumbnail_url: '/c' },
]
store.dateGroups = []
const w = mountGrid(pinia)
expect(w.findAll('.gi').length).toBe(3)
})
it('renders grouped by date when date groups are present', () => {
const pinia = freshPinia()
const store = useGalleryStore()
store.images = [
{ id: 1, thumbnail_url: '/a' },
{ id: 2, thumbnail_url: '/b' },
]
store.dateGroups = [{ year: 2026, month: 6, image_ids: [1, 2] }]
const w = mountGrid(pinia)
expect(w.find('.fc-gallery-grid__date-header').exists()).toBe(true)
expect(w.findAll('.gi').length).toBe(2)
})
})
+23
View File
@@ -435,3 +435,26 @@ async def test_trigger_vacuum_queues_the_task(client, monkeypatch):
resp = await client.post("/api/admin/maintenance/vacuum")
assert resp.status_code == 202
assert calls == [1]
# --- Tier-A: POST /tags/reset-content -------------------------------
@pytest.mark.asyncio
async def test_reset_content_tagging_dry_run_returns_counts(client, db):
db.add_all([
Tag(name="solo", kind=TagKind.general),
Tag(name="naruto", kind=TagKind.character),
Tag(name="Naruto", kind=TagKind.fandom),
Tag(name="my-series", kind=TagKind.series),
])
await db.commit()
resp = await client.post(
"/api/admin/tags/reset-content", json={"dry_run": True}
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["count"] == 2
assert body["by_kind"] == {"general": 1, "character": 1}
# dry-run leaves the rows in place — fandom + series untouched too.
assert "deleted" not in body
+21
View File
@@ -155,6 +155,27 @@ async def test_audit_history_returns_recent_runs(client, db):
assert len(body["runs"]) >= 3
@pytest.mark.asyncio
async def test_audit_history_filters_by_rule(client, db):
db.add(LibraryAuditRun(
rule="transparency", params={"threshold": 0.9},
status="applied", matched_ids=[], finished_at=datetime.now(UTC),
))
db.add(LibraryAuditRun(
rule="single_color", params={"threshold": 0.95, "tolerance": 30},
status="ready", matched_count=2, matched_ids=[1, 2],
finished_at=datetime.now(UTC),
))
await db.commit()
# ?rule=&limit=1 → just THIS rule's latest run (the card-reconnect query).
resp = await client.get("/api/cleanup/audit?rule=single_color&limit=1")
assert resp.status_code == 200
runs = (await resp.get_json())["runs"]
assert len(runs) == 1
assert runs[0]["rule"] == "single_color"
assert runs[0]["matched_count"] == 2
@pytest.mark.asyncio
async def test_audit_apply_with_token_deletes(client, db, tmp_path):
rec = await _seed_image(db, tmp_path, w=100, h=100, name="apply.png")
+60
View File
@@ -9,6 +9,7 @@ import pytest
from sqlalchemy import func, select
from backend.app.models import Artist, ImageRecord, Tag, TagKind
from backend.app.models.series_page import SeriesPage
from backend.app.models.tag import image_tag
from backend.app.services import cleanup_service
@@ -333,3 +334,62 @@ def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
assert "kept" in surviving_names
assert "bye" not in surviving_names
# --- reset_content_tagging ------------------------------------------
def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path):
a = _make_artist(db_sync, slug="rc")
img = _make_image(
db_sync, artist=a, path=str(tmp_path / "r.jpg"), sha256="1" * 64,
)
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
_make_tag(db_sync, name="my-series", kind=TagKind.series)
db_sync.execute(image_tag.insert().values([
{"image_record_id": img.id, "tag_id": g.id},
{"image_record_id": img.id, "tag_id": c.id},
]))
db_sync.commit()
result = cleanup_service.reset_content_tagging(db_sync, dry_run=True)
assert result["count"] == 2
assert result["by_kind"] == {"general": 1, "character": 1}
assert result["applications"] == 2
# Nothing deleted — all 4 tags still present.
assert db_sync.execute(select(func.count(Tag.id))).scalar_one() == 4
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
a = _make_artist(db_sync, slug="rc2")
img = _make_image(
db_sync, artist=a, path=str(tmp_path / "r2.jpg"), sha256="2" * 64,
)
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
s = _make_tag(db_sync, name="my-series", kind=TagKind.series)
db_sync.execute(image_tag.insert().values([
{"image_record_id": img.id, "tag_id": g.id},
{"image_record_id": img.id, "tag_id": c.id},
{"image_record_id": img.id, "tag_id": s.id}, # series membership
]))
db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1))
db_sync.commit()
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
assert result["deleted"] == 2
# general + character gone; fandom + series kept.
kinds = db_sync.execute(select(Tag.kind)).scalars().all()
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
assert kind_vals == {"fandom", "series"}
# series_page ordering survived.
assert db_sync.execute(
select(func.count()).select_from(SeriesPage)
).scalar_one() == 1
# Only the series image_tag association survived (content ones cascaded).
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
assert remaining == [s.id]