Merge pull request 'dev→main: similar-search render fix + reset-content-tagging + scan persistence' (#66) from dev into main
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 2s
Build images / build-ml (push) Successful in 9s
CI / backend-lint-and-test (push) Successful in 13s
Build images / build-web (push) Successful in 11s
CI / frontend-build (push) Successful in 22s
CI / integration (push) Successful in 2m58s
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 2s
Build images / build-ml (push) Successful in 9s
CI / backend-lint-and-test (push) Successful in 13s
Build images / build-web (push) Successful in 11s
CI / frontend-build (push) Successful in 22s
CI / integration (push) Successful in 2m58s
This commit was merged in pull request #66.
This commit is contained in:
@@ -224,6 +224,27 @@ async def tags_purge_legacy():
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/reset-content", methods=["POST"])
|
||||
async def tags_reset_content():
|
||||
"""Tier-A: delete ALL general + character tags (the Camie-suggestable
|
||||
content vocabulary) so the operator can re-tag from scratch via
|
||||
auto-suggest. fandom + series tags + series_page ordering are preserved,
|
||||
and image tagger_predictions are untouched so suggestions repopulate.
|
||||
dry-run preview returns per-kind counts + applications + a sample so the
|
||||
UI shows exactly what'll go before the operator confirms (dry_run=false).
|
||||
Irreversible except via DB backup restore."""
|
||||
from ..services.cleanup_service import reset_content_tagging
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda sync_sess: reset_content_tagging(sync_sess, dry_run=dry_run)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
|
||||
async def db_stats():
|
||||
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
|
||||
|
||||
@@ -154,12 +154,15 @@ async def audit_history():
|
||||
limit = min(int(request.args.get("limit", "20")), 100)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
# Optional rule filter so a card can reconnect to ITS latest run on mount
|
||||
# (?rule=transparency&limit=1) — the audit survives navigation; the UI
|
||||
# rehydrates from this rather than losing the in-flight scan.
|
||||
rule = request.args.get("rule") or None
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LibraryAuditRun)
|
||||
.order_by(LibraryAuditRun.id.desc())
|
||||
.limit(limit)
|
||||
)).scalars().all()
|
||||
stmt = select(LibraryAuditRun).order_by(LibraryAuditRun.id.desc())
|
||||
if rule is not None:
|
||||
stmt = stmt.where(LibraryAuditRun.rule == rule)
|
||||
rows = (await session.execute(stmt.limit(limit))).scalars().all()
|
||||
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
|
||||
|
||||
|
||||
|
||||
@@ -455,6 +455,62 @@ def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
# The Camie-suggestable CONTENT vocabulary. "Reset content tagging" wipes
|
||||
# these so the operator can re-tag from scratch via auto-suggest. fandom +
|
||||
# series (and series_page ordering) are deliberately NOT here — they're kept.
|
||||
RESETTABLE_TAG_KINDS = ("general", "character")
|
||||
|
||||
|
||||
def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
|
||||
"""Count (dry_run) or DELETE every general + character tag so the operator
|
||||
can re-tag from scratch via the Camie auto-suggest.
|
||||
|
||||
PRESERVED: fandom + series tags and their series_page ordering, plus every
|
||||
image's image_record.tagger_predictions (untouched) so suggestions
|
||||
repopulate immediately. CASCADE on image_tag / tag_alias / tag_allowlist /
|
||||
tag_reference_embedding / tag_suggestion_rejection clears each deleted
|
||||
tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting
|
||||
character tags never touches the fandom rows. Irreversible except via DB
|
||||
backup restore.
|
||||
|
||||
Returns:
|
||||
{"by_kind": {"general": N, "character": M},
|
||||
"count": total tags,
|
||||
"applications": image_tag rows that will be / were removed,
|
||||
"sample_names": [first 50],
|
||||
and on live runs "deleted": total}
|
||||
"""
|
||||
predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
|
||||
rows = session.execute(
|
||||
select(Tag.id, Tag.name, Tag.kind).where(predicate)
|
||||
).all()
|
||||
by_kind: dict[str, int] = {}
|
||||
for _id, _name, kind in rows:
|
||||
key = kind.value if hasattr(kind, "value") else str(kind)
|
||||
by_kind[key] = by_kind.get(key, 0) + 1
|
||||
# Headline impact: applications (image_tag rows) that vanish via cascade.
|
||||
applications = session.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.where(image_tag.c.tag_id.in_(select(Tag.id).where(predicate)))
|
||||
).scalar_one()
|
||||
sample = [name for _id, name, _kind in rows[:50]]
|
||||
total = len(rows)
|
||||
result = {
|
||||
"by_kind": by_kind,
|
||||
"count": total,
|
||||
"applications": applications,
|
||||
"sample_names": sample,
|
||||
}
|
||||
if dry_run:
|
||||
return result
|
||||
if total:
|
||||
session.execute(Tag.__table__.delete().where(predicate))
|
||||
session.commit()
|
||||
result["deleted"] = total
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -112,6 +112,16 @@ onMounted(async () => {
|
||||
await store.loadDefaults()
|
||||
threshold.value = store.defaults.single_color_threshold
|
||||
tolerance.value = store.defaults.single_color_tolerance
|
||||
// Reconnect to this rule's latest run so a scan started before navigating
|
||||
// away keeps showing progress / its result on return (the scan itself runs
|
||||
// backend-side regardless).
|
||||
try {
|
||||
const latest = await store.latestAuditForRule('single_color')
|
||||
if (latest) {
|
||||
audit.value = latest
|
||||
if (latest.status === 'running') startPoll(latest.id)
|
||||
}
|
||||
} catch { /* non-fatal — card still works for a fresh scan */ }
|
||||
})
|
||||
|
||||
onUnmounted(() => stopPoll())
|
||||
|
||||
@@ -97,6 +97,16 @@ let pollTimer = null
|
||||
onMounted(async () => {
|
||||
await store.loadDefaults()
|
||||
threshold.value = store.defaults.transparency_threshold
|
||||
// Reconnect to this rule's latest run so a scan started before navigating
|
||||
// away keeps showing progress / its result on return (the scan itself runs
|
||||
// backend-side regardless).
|
||||
try {
|
||||
const latest = await store.latestAuditForRule('transparency')
|
||||
if (latest) {
|
||||
audit.value = latest
|
||||
if (latest.status === 'running') startPoll(latest.id)
|
||||
}
|
||||
} catch { /* non-fatal — card still works for a fresh scan */ }
|
||||
})
|
||||
|
||||
onUnmounted(() => stopPoll())
|
||||
|
||||
@@ -15,6 +15,19 @@
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- Similar-mode (and any non-date-grouped result set) returns no date
|
||||
groups — the results are ranked, not chronological. Render them as a
|
||||
single flat list in their given order rather than nothing. -->
|
||||
<div
|
||||
v-if="!store.dateGroups.length && store.images.length"
|
||||
class="fc-gallery-grid__items"
|
||||
>
|
||||
<GalleryItem
|
||||
v-for="img in store.images"
|
||||
:key="img.id" :image="img" @open="$emit('open', img.id)"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div v-if="store.loading" class="fc-gallery-grid__sentinel">
|
||||
<v-progress-circular indeterminate color="accent" size="28" />
|
||||
</div>
|
||||
|
||||
@@ -89,6 +89,53 @@
|
||||
@click="onKindCommit"
|
||||
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
|
||||
</div>
|
||||
|
||||
<v-divider class="my-5" />
|
||||
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong class="text-error">Reset content tagging.</strong>
|
||||
Deletes every <code>general</code> and <code>character</code> tag and
|
||||
removes them from every image, so you can re-tag from scratch with the
|
||||
auto-suggest. <strong>Fandoms and series (with their page order) are
|
||||
kept</strong>, and each image's saved predictions are untouched — open
|
||||
an image and its suggestions reappear.
|
||||
</p>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
Irreversible — there's no undo except restoring a DB backup.
|
||||
Back one up first (Settings → Maintenance → Backup).
|
||||
</v-alert>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingResetPreview"
|
||||
class="mb-3"
|
||||
@click="onResetPreview"
|
||||
>Preview content-tag reset</v-btn>
|
||||
|
||||
<div v-if="resetPreview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ resetPreview.count }}</strong> content tag(s)
|
||||
<span v-for="(n, k) in resetPreview.by_kind" :key="k" class="fc-muted">
|
||||
({{ k }}: {{ n }})
|
||||
</span>
|
||||
across <strong>{{ resetPreview.applications }}</strong> image
|
||||
application(s).
|
||||
</p>
|
||||
<div v-if="resetPreview.sample_names?.length" class="fc-name-grid mb-3">
|
||||
<span v-for="n in resetPreview.sample_names" :key="n" class="fc-name">
|
||||
{{ n }}
|
||||
</span>
|
||||
</div>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-delete-alert"
|
||||
:disabled="!resetPreview.count"
|
||||
:loading="resetCommitting"
|
||||
@click="onResetCommit"
|
||||
>Delete {{ resetPreview.count }} content tag(s) +
|
||||
{{ resetPreview.applications }} application(s)</v-btn>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
@@ -105,6 +152,9 @@ const committing = ref(false)
|
||||
const kindPreview = ref(null)
|
||||
const loadingKindPreview = ref(false)
|
||||
const kindCommitting = ref(false)
|
||||
const resetPreview = ref(null)
|
||||
const loadingResetPreview = ref(false)
|
||||
const resetCommitting = ref(false)
|
||||
|
||||
async function onPreview() {
|
||||
loadingPreview.value = true
|
||||
@@ -143,6 +193,25 @@ async function onKindCommit() {
|
||||
kindCommitting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onResetPreview() {
|
||||
loadingResetPreview.value = true
|
||||
try {
|
||||
resetPreview.value = await store.resetContentTagging({ dryRun: true })
|
||||
} finally {
|
||||
loadingResetPreview.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onResetCommit() {
|
||||
resetCommitting.value = true
|
||||
try {
|
||||
await store.resetContentTagging({ dryRun: false })
|
||||
resetPreview.value = { count: 0, by_kind: {}, applications: 0, sample_names: [] }
|
||||
} finally {
|
||||
resetCommitting.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -127,6 +127,21 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Destructive: deletes ALL general + character tags so the operator can
|
||||
// re-tag from scratch via auto-suggest. fandom + series preserved.
|
||||
async function resetContentTagging({ dryRun = true } = {}) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/tags/reset-content',
|
||||
{ body: { dry_run: dryRun } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||
|
||||
/**
|
||||
@@ -162,6 +177,7 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
tagUsageCount,
|
||||
pruneUnusedTags,
|
||||
purgeLegacyTags,
|
||||
resetContentTagging,
|
||||
pollTaskUntilDone,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -55,6 +55,14 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
return body.runs
|
||||
}
|
||||
|
||||
// The most recent audit run for a given rule, or null. Cards call this on
|
||||
// mount to reconnect to a scan that's still running (or to show the last
|
||||
// completed result) after the user navigates away and back.
|
||||
async function latestAuditForRule(rule) {
|
||||
const body = await api.get('/api/cleanup/audit', { params: { rule, limit: 1 } })
|
||||
return (body.runs && body.runs[0]) || null
|
||||
}
|
||||
|
||||
async function applyAudit(id, confirm) {
|
||||
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
|
||||
}
|
||||
@@ -67,6 +75,6 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
defaults, recentRuns,
|
||||
loadDefaults,
|
||||
previewMinDim, deleteMinDim,
|
||||
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
|
||||
startAudit, getAudit, loadHistory, latestAuditForRule, applyAudit, cancelAudit,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { mount } from '@vue/test-utils'
|
||||
|
||||
import GalleryGrid from '../../src/components/gallery/GalleryGrid.vue'
|
||||
import { useGalleryStore } from '../../src/stores/gallery.js'
|
||||
import { freshPinia } from '../support/mountComponent.js'
|
||||
|
||||
const GIStub = { name: 'GalleryItem', props: ['image'], template: '<div class="gi" />' }
|
||||
|
||||
function mountGrid(pinia) {
|
||||
return mount(GalleryGrid, {
|
||||
global: {
|
||||
plugins: [pinia],
|
||||
stubs: { GalleryItem: GIStub, RouterLink: { template: '<a><slot /></a>' } },
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('GalleryGrid', () => {
|
||||
it('renders a flat ranked list when there are no date groups (similar mode)', () => {
|
||||
const pinia = freshPinia()
|
||||
const store = useGalleryStore()
|
||||
// similar-mode shape: images present, date_groups empty, no cursor.
|
||||
store.images = [
|
||||
{ id: 1, thumbnail_url: '/a' },
|
||||
{ id: 2, thumbnail_url: '/b' },
|
||||
{ id: 3, thumbnail_url: '/c' },
|
||||
]
|
||||
store.dateGroups = []
|
||||
const w = mountGrid(pinia)
|
||||
expect(w.findAll('.gi').length).toBe(3)
|
||||
})
|
||||
|
||||
it('renders grouped by date when date groups are present', () => {
|
||||
const pinia = freshPinia()
|
||||
const store = useGalleryStore()
|
||||
store.images = [
|
||||
{ id: 1, thumbnail_url: '/a' },
|
||||
{ id: 2, thumbnail_url: '/b' },
|
||||
]
|
||||
store.dateGroups = [{ year: 2026, month: 6, image_ids: [1, 2] }]
|
||||
const w = mountGrid(pinia)
|
||||
expect(w.find('.fc-gallery-grid__date-header').exists()).toBe(true)
|
||||
expect(w.findAll('.gi').length).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -435,3 +435,26 @@ async def test_trigger_vacuum_queues_the_task(client, monkeypatch):
|
||||
resp = await client.post("/api/admin/maintenance/vacuum")
|
||||
assert resp.status_code == 202
|
||||
assert calls == [1]
|
||||
|
||||
|
||||
# --- Tier-A: POST /tags/reset-content -------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_content_tagging_dry_run_returns_counts(client, db):
|
||||
db.add_all([
|
||||
Tag(name="solo", kind=TagKind.general),
|
||||
Tag(name="naruto", kind=TagKind.character),
|
||||
Tag(name="Naruto", kind=TagKind.fandom),
|
||||
Tag(name="my-series", kind=TagKind.series),
|
||||
])
|
||||
await db.commit()
|
||||
resp = await client.post(
|
||||
"/api/admin/tags/reset-content", json={"dry_run": True}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["count"] == 2
|
||||
assert body["by_kind"] == {"general": 1, "character": 1}
|
||||
# dry-run leaves the rows in place — fandom + series untouched too.
|
||||
assert "deleted" not in body
|
||||
|
||||
@@ -155,6 +155,27 @@ async def test_audit_history_returns_recent_runs(client, db):
|
||||
assert len(body["runs"]) >= 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audit_history_filters_by_rule(client, db):
|
||||
db.add(LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.9},
|
||||
status="applied", matched_ids=[], finished_at=datetime.now(UTC),
|
||||
))
|
||||
db.add(LibraryAuditRun(
|
||||
rule="single_color", params={"threshold": 0.95, "tolerance": 30},
|
||||
status="ready", matched_count=2, matched_ids=[1, 2],
|
||||
finished_at=datetime.now(UTC),
|
||||
))
|
||||
await db.commit()
|
||||
# ?rule=&limit=1 → just THIS rule's latest run (the card-reconnect query).
|
||||
resp = await client.get("/api/cleanup/audit?rule=single_color&limit=1")
|
||||
assert resp.status_code == 200
|
||||
runs = (await resp.get_json())["runs"]
|
||||
assert len(runs) == 1
|
||||
assert runs[0]["rule"] == "single_color"
|
||||
assert runs[0]["matched_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audit_apply_with_token_deletes(client, db, tmp_path):
|
||||
rec = await _seed_image(db, tmp_path, w=100, h=100, name="apply.png")
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Tag, TagKind
|
||||
from backend.app.models.series_page import SeriesPage
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services import cleanup_service
|
||||
|
||||
@@ -333,3 +334,62 @@ def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
|
||||
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
|
||||
assert "kept" in surviving_names
|
||||
assert "bye" not in surviving_names
|
||||
|
||||
|
||||
# --- reset_content_tagging ------------------------------------------
|
||||
|
||||
|
||||
def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="rc")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "r.jpg"), sha256="1" * 64,
|
||||
)
|
||||
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
||||
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
||||
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
||||
_make_tag(db_sync, name="my-series", kind=TagKind.series)
|
||||
db_sync.execute(image_tag.insert().values([
|
||||
{"image_record_id": img.id, "tag_id": g.id},
|
||||
{"image_record_id": img.id, "tag_id": c.id},
|
||||
]))
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.reset_content_tagging(db_sync, dry_run=True)
|
||||
assert result["count"] == 2
|
||||
assert result["by_kind"] == {"general": 1, "character": 1}
|
||||
assert result["applications"] == 2
|
||||
# Nothing deleted — all 4 tags still present.
|
||||
assert db_sync.execute(select(func.count(Tag.id))).scalar_one() == 4
|
||||
|
||||
|
||||
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="rc2")
|
||||
img = _make_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "r2.jpg"), sha256="2" * 64,
|
||||
)
|
||||
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
||||
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
||||
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
||||
s = _make_tag(db_sync, name="my-series", kind=TagKind.series)
|
||||
db_sync.execute(image_tag.insert().values([
|
||||
{"image_record_id": img.id, "tag_id": g.id},
|
||||
{"image_record_id": img.id, "tag_id": c.id},
|
||||
{"image_record_id": img.id, "tag_id": s.id}, # series membership
|
||||
]))
|
||||
db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1))
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
|
||||
assert result["deleted"] == 2
|
||||
|
||||
# general + character gone; fandom + series kept.
|
||||
kinds = db_sync.execute(select(Tag.kind)).scalars().all()
|
||||
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
|
||||
assert kind_vals == {"fandom", "series"}
|
||||
# series_page ordering survived.
|
||||
assert db_sync.execute(
|
||||
select(func.count()).select_from(SeriesPage)
|
||||
).scalar_one() == 1
|
||||
# Only the series image_tag association survived (content ones cascaded).
|
||||
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
|
||||
assert remaining == [s.id]
|
||||
|
||||
Reference in New Issue
Block a user