feat(ia): wave 1 — Import tab dissolves, Maintenance regroups by system, one extension home
Settings IA per the approved A3 design (the old layout was the two-app merge fossilized): - Import tab retired: ImportTriggerPanel + ImportTaskList deleted (manual /import scans stay API-level; imports arrive via downloads/extension, heal via the Layer-2 auto-refetch sweep, and show in Activity). ImportFiltersForm moves to Maintenance → 'Ingestion & filters' and loads its own settings; the import store shrinks to settings-only (no remaining consumers of the scan/task-list machinery). Overview's pending banner now points at Activity. - Maintenance regrouped: Ingestion & filters / GPU agent & embeddings (GpuAgent, Failed processing, CPU embedding backfill) / Tagging (sliders, Heads, Aliases) / Library health (MissingFiles, Thumbnails, DB, Archive re-extract demoted last) / Storage. - One extension home: BrowserExtensionCard moves from Settings → Overview to Subscriptions → Settings, above the API key bar it authenticates. - Single-color import filter WIRED: skip_single_color/threshold existed since FC-2 but nothing read them (the audit module's docstring said as much) — now enforced on both import paths via the audit's canonical predicate (tolerance 30, matching the Cleanup card default; animated images exempt like the transparency check). Default stays off; test added. - Dead weight: PlaceholderView (zero refs) and the permanently-disabled 'Export failed logs (CSV — v2)' menu stub deleted; stale docs fixed (celery queue docstring, threshold comment citing retired tasks, ml package docstring, HeadsCard 'replaces Camie' blurb). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -7,7 +7,7 @@ Queues:
|
||||
download — gallery-dl tasks (FC-3)
|
||||
scan — periodic source checks (FC-3) — kept separate so long imports
|
||||
don't starve the scheduler
|
||||
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
||||
maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
|
||||
default — anything not explicitly routed
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Single-color audit: matches images where one color dominates beyond
|
||||
the threshold (within the given Euclidean RGB tolerance). The first
|
||||
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||
future spec can adopt it on the import path too.
|
||||
the threshold (within the given Euclidean RGB tolerance). The canonical
|
||||
predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||
SkipReason.single_color), so what the audit flags and what the import
|
||||
skips can never disagree.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -44,6 +44,7 @@ from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from ..utils.slug import slugify
|
||||
from .archive_extractor import extract_archive, is_archive
|
||||
from .attachment_store import AttachmentStore
|
||||
from .audits import single_color
|
||||
from .link_extract import extract_external_links
|
||||
from .thumbnailer import Thumbnailer
|
||||
|
||||
@@ -790,6 +791,13 @@ class Importer:
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(source):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
|
||||
# Artist anchored to the attribution path (folder→artist), resolved
|
||||
# UP-FRONT so the enrich-on-duplicate branches link provenance with the
|
||||
# right artist even when the sidecar carries none — which is now the norm
|
||||
@@ -1123,6 +1131,13 @@ class Importer:
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(path):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
else:
|
||||
# Best-effort probe for dims + duration so downloaded videos can dedup
|
||||
# (#871). LENIENT: unlike _import_media this path does not reject on a
|
||||
@@ -1538,6 +1553,33 @@ class Importer:
|
||||
# Benign orphan; the DB swap already committed. Don't undo it.
|
||||
pass
|
||||
|
||||
# Matches the Cleanup audit card's default tolerance: the import-side
|
||||
# filter and the retroactive audit must agree on what "single color" MEANS
|
||||
# (Euclidean RGB distance to the dominant color); only the match threshold
|
||||
# is operator-tunable per surface.
|
||||
_SINGLE_COLOR_TOLERANCE = 30
|
||||
|
||||
def _single_color_hit(self, source: Path) -> bool:
|
||||
"""True when one color dominates beyond the configured threshold — the
|
||||
same canonical predicate the Cleanup audit runs (audits.single_color,
|
||||
whose docstring anticipated this adoption; the skip_single_color
|
||||
setting existed but was never wired until 2026-07-02). Never raises:
|
||||
unreadable files were already rejected by verify() upstream, and a
|
||||
residual decode error just declines to match (the import proceeds)."""
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
if getattr(im, "is_animated", False):
|
||||
# Frame 0 only would misjudge animations; skip like the
|
||||
# transparency check does.
|
||||
return False
|
||||
return single_color.evaluate(
|
||||
im,
|
||||
threshold=self.settings.single_color_threshold,
|
||||
tolerance=self._SINGLE_COLOR_TOLERANCE,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _transparency_pct(self, source: Path) -> float:
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases."""
|
||||
"""ML pipeline services: embedders, heads (the learning suggester), suggestions,
|
||||
GPU-job queue + failure triage, CCIP characters, crops/regions, allowlist and
|
||||
aliases."""
|
||||
|
||||
@@ -139,10 +139,9 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"download": 30,
|
||||
# Audit 2026-06-02 — maintenance/scan queues run tasks that
|
||||
# legitimately exceed the 5-min default (verify_integrity at 70m
|
||||
# hard, scan_directory at 70m hard, apply_allowlist_tags /
|
||||
# recompute_centroids / backfill_phash at 35m hard). 75 min lives
|
||||
# above the longest of those and the per-task overrides below
|
||||
# cover the outliers (backups, library audit).
|
||||
# hard, scan_directory at 70m hard, backfill_phash at 35m hard).
|
||||
# 75 min lives above the longest of those and the per-task
|
||||
# overrides below cover the outliers (backups, library audit).
|
||||
"maintenance": 75,
|
||||
"scan": 75,
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<MaintenanceTile
|
||||
icon="mdi-brain"
|
||||
title="Concept heads (the learning suggester)"
|
||||
blurb="Train the per-concept heads that turn your tags into suggestions — they replace Camie and sharpen every time you accept or reject."
|
||||
blurb="Train the per-concept heads that turn your tags into suggestions — they learn from your library and sharpen every time you accept or reject."
|
||||
:open="running"
|
||||
>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
|
||||
@@ -90,10 +90,14 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { reactive, watch } from 'vue'
|
||||
import { onMounted, reactive, watch } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
// Self-sufficient since the Import tab dissolved (2026-07-02): this form now
|
||||
// lives in Maintenance → Ingestion & filters and loads its own settings
|
||||
// instead of relying on the old tab's mount hook.
|
||||
onMounted(() => { if (!store.settings) store.loadSettings() })
|
||||
// Labelled stops so the less-initiated get the gist without knowing what a
|
||||
// Hamming distance is. 0 = byte-for-byte only; 10 = the shipped default.
|
||||
const PHASH_TICKS = { 0: 'Exact', 4: 'Strict', 10: 'Default', 16: 'Loose' }
|
||||
|
||||
@@ -1,254 +0,0 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<CardHeading title="Recent import tasks">
|
||||
<v-spacer />
|
||||
<v-select
|
||||
v-model="statusFilter" :items="statusOptions" density="compact"
|
||||
hide-details style="max-width: 180px;" @update:model-value="onFilterChange"
|
||||
/>
|
||||
<v-btn variant="text" rounded="pill" size="small" @click="onRefresh">
|
||||
<v-icon start>mdi-refresh</v-icon>
|
||||
Refresh
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:disabled="!hasFailed" @click="onRetryFailed"
|
||||
>
|
||||
Retry failed
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:disabled="!hasStuck" @click="onClearStuckOpen"
|
||||
>
|
||||
Clear stuck…
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="error"
|
||||
@click="onClearOpen"
|
||||
>
|
||||
Clear completed…
|
||||
</v-btn>
|
||||
</CardHeading>
|
||||
<v-data-table-virtual
|
||||
:headers="headers" :items="store.tasks" :loading="store.tasksLoading"
|
||||
height="480" density="compact" fixed-header no-data-text="No tasks yet — trigger a scan above."
|
||||
>
|
||||
<template #item.status="{ item }">
|
||||
<v-chip :color="statusColor(item.status)" size="small" variant="tonal">
|
||||
{{ item.status }}
|
||||
</v-chip>
|
||||
</template>
|
||||
<template #item.source_path="{ item }">
|
||||
<span :title="item.source_path">{{ shorten(item.source_path) }}</span>
|
||||
</template>
|
||||
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
||||
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
||||
<template #item.error="{ item }">
|
||||
<button
|
||||
v-if="item.error" type="button" class="fc-err-link text-caption"
|
||||
@click="openError(`Task ${item.id} failed`, item.error)"
|
||||
title="Click for full error"
|
||||
>{{ shorten(item.error, 60) }}</button>
|
||||
</template>
|
||||
<template #item.actions="{ item }">
|
||||
<v-btn
|
||||
v-if="item.status === 'failed'"
|
||||
icon size="x-small" variant="text"
|
||||
:loading="refetching === item.id"
|
||||
@click="onRefetch(item)"
|
||||
>
|
||||
<v-icon size="small">mdi-cloud-refresh</v-icon>
|
||||
<v-tooltip activator="parent" location="top">
|
||||
Re-fetch original (re-download from source)
|
||||
</v-tooltip>
|
||||
</v-btn>
|
||||
</template>
|
||||
</v-data-table-virtual>
|
||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
||||
</div>
|
||||
|
||||
<v-dialog v-model="clearDialog" max-width="400">
|
||||
<v-card>
|
||||
<v-card-title>Clear completed tasks</v-card-title>
|
||||
<v-card-text>
|
||||
<v-select
|
||||
v-model="clearAgeDays" label="Older than"
|
||||
:items="[
|
||||
{ title: 'All finished', value: 0 },
|
||||
{ title: '1 day', value: 1 },
|
||||
{ title: '7 days', value: 7 },
|
||||
{ title: '30 days', value: 30 }
|
||||
]"
|
||||
/>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn @click="clearDialog = false">Cancel</v-btn>
|
||||
<v-btn color="error" rounded="pill" @click="onClearConfirm">Clear</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<v-dialog v-model="clearStuckDialog" max-width="480">
|
||||
<v-card>
|
||||
<v-card-title>Clear stuck tasks</v-card-title>
|
||||
<v-card-text>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
Force every <strong>pending / queued / processing</strong> task to
|
||||
<strong>failed</strong> and finalize any active batch that
|
||||
has no remaining work. Use this when the automatic recovery
|
||||
sweep keeps re-queueing the same row (e.g., corrupt file in
|
||||
an autoretry loop, or worker model missing).
|
||||
</v-alert>
|
||||
<p class="text-body-2">
|
||||
Tasks remain in the database with status=<code>failed</code>;
|
||||
click <em>Retry failed</em> once the underlying cause is
|
||||
resolved to re-queue them.
|
||||
</p>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn @click="clearStuckDialog = false">Cancel</v-btn>
|
||||
<v-btn color="warning" rounded="pill" @click="onClearStuckConfirm">Clear stuck</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<ErrorDetailModal
|
||||
v-model="showErrorModal"
|
||||
:title="errorModalTitle"
|
||||
:message="errorModalMessage"
|
||||
/>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import { computed, ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
|
||||
const store = useImportStore()
|
||||
const statusFilter = ref(null)
|
||||
const clearDialog = ref(false)
|
||||
// Click-to-open modal for full error text (operator-flagged 2026-05-26
|
||||
// — the prior :title="..." tooltip cramped multi-line SQLAlchemy
|
||||
// tracebacks into an unusable popup with no copy-paste affordance).
|
||||
const showErrorModal = ref(false)
|
||||
const errorModalTitle = ref('')
|
||||
const errorModalMessage = ref('')
|
||||
|
||||
function openError(title, message) {
|
||||
errorModalTitle.value = title
|
||||
errorModalMessage.value = message || ''
|
||||
showErrorModal.value = true
|
||||
}
|
||||
const clearAgeDays = ref(7)
|
||||
const clearStuckDialog = ref(false)
|
||||
|
||||
const statusOptions = [
|
||||
{ title: 'All', value: null },
|
||||
{ title: 'Pending', value: 'pending' },
|
||||
{ title: 'Queued', value: 'queued' },
|
||||
{ title: 'Processing', value: 'processing' },
|
||||
{ title: 'Complete', value: 'complete' },
|
||||
{ title: 'Skipped', value: 'skipped' },
|
||||
{ title: 'Failed', value: 'failed' }
|
||||
]
|
||||
|
||||
const headers = [
|
||||
{ title: 'Status', key: 'status', sortable: false, width: 120 },
|
||||
{ title: 'Source', key: 'source_path', sortable: false },
|
||||
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
||||
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
||||
{ title: 'Note', key: 'error', sortable: false },
|
||||
{ title: '', key: 'actions', sortable: false, width: 56 }
|
||||
]
|
||||
|
||||
const refetching = ref(null)
|
||||
const _REFETCH_MSG = {
|
||||
refetch_queued: { text: 'Re-fetch queued — re-downloading from source', type: 'success' },
|
||||
no_source: { text: 'No re-fetchable source (filesystem import — replace the file manually)', type: 'info' },
|
||||
already_refetched: { text: 'Already re-fetched once', type: 'info' },
|
||||
}
|
||||
async function onRefetch(item) {
|
||||
refetching.value = item.id
|
||||
try {
|
||||
const res = await store.refetchTask(item.id)
|
||||
const msg = _REFETCH_MSG[res.status] || { text: `Re-fetch: ${res.status}`, type: 'info' }
|
||||
toast(msg)
|
||||
} catch (e) {
|
||||
toast({ text: `Re-fetch failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
refetching.value = null
|
||||
}
|
||||
}
|
||||
|
||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||
const hasStuck = computed(() => store.tasks.some(
|
||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||
))
|
||||
|
||||
function statusColor(s) {
|
||||
return {
|
||||
complete: 'success',
|
||||
skipped: 'warning',
|
||||
failed: 'error',
|
||||
processing: 'accent',
|
||||
queued: 'info',
|
||||
pending: 'info'
|
||||
}[s] || 'default'
|
||||
}
|
||||
function shorten(s, max = 90) {
|
||||
if (!s) return ''
|
||||
if (s.length <= max) return s
|
||||
const head = Math.floor((max - 3) * 0.6)
|
||||
const tail = max - 3 - head
|
||||
return s.slice(0, head) + '...' + s.slice(-tail)
|
||||
}
|
||||
function formatBytes(b) {
|
||||
if (!b) return ''
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB']
|
||||
let i = 0; let v = b
|
||||
while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ }
|
||||
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
||||
}
|
||||
function formatDate(s) {
|
||||
try { return new Date(s).toLocaleString() } catch { return s }
|
||||
}
|
||||
|
||||
async function onRefresh() { await store.loadTasks(true) }
|
||||
function onFilterChange() { store.setStatusFilter(statusFilter.value); store.loadTasks(true) }
|
||||
async function onLoadMore() { await store.loadTasks(false) }
|
||||
async function onRetryFailed() { await store.retryFailed() }
|
||||
function onClearOpen() { clearDialog.value = true }
|
||||
async function onClearConfirm() {
|
||||
await store.clearCompleted(clearAgeDays.value)
|
||||
clearDialog.value = false
|
||||
}
|
||||
function onClearStuckOpen() { clearStuckDialog.value = true }
|
||||
async function onClearStuckConfirm() {
|
||||
await store.clearStuck()
|
||||
clearStuckDialog.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-err-link {
|
||||
/* Truncated error preview as a clickable button — opens
|
||||
ErrorDetailModal with the full text. Inherits the row's font
|
||||
sizing so it doesn't visually drift from the prior tooltip-bearing
|
||||
span. */
|
||||
color: rgb(var(--v-theme-error, 220 80 80));
|
||||
background: transparent;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
font: inherit;
|
||||
text-align: left;
|
||||
text-decoration: underline dotted;
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-err-link:hover { text-decoration: underline; }
|
||||
</style>
|
||||
@@ -1,97 +0,0 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<v-card-title>Trigger scan</v-card-title>
|
||||
<v-card-text>
|
||||
<div v-if="store.activeBatch" class="d-flex align-center mb-3" style="gap: 12px;">
|
||||
<v-progress-circular
|
||||
indeterminate color="accent" size="20"
|
||||
/>
|
||||
<span>
|
||||
{{ store.activeBatch.scan_mode === 'deep' ? 'Deep scanning' : 'Scanning' }}
|
||||
{{ store.activeBatch.source_path || '/import' }} —
|
||||
imported {{ store.activeBatch.imported }},
|
||||
<template v-if="store.activeBatch.scan_mode === 'deep'">
|
||||
refreshed {{ store.activeBatch.refreshed || 0 }},
|
||||
</template>
|
||||
skipped {{ store.activeBatch.skipped }},
|
||||
failed {{ store.activeBatch.failed }} /
|
||||
{{ store.activeBatch.total_files }} files
|
||||
</span>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:loading="clearing" @click="onClearStuck"
|
||||
>
|
||||
Clear stuck
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<p class="text-body-2 mb-3">
|
||||
<span v-if="!store.activeBatch">
|
||||
<strong>Quick scan</strong> walks <code>/import</code> and enqueues
|
||||
new files only.
|
||||
<strong>Deep scan</strong> additionally re-walks already-imported
|
||||
files so updated sidecar metadata (post title/date/attribution) and
|
||||
previously-NULL phashes / artist links get refreshed. Use after
|
||||
bulk-downloading fresh sidecars for existing content. Both modes
|
||||
route non-media + sidecar pairs through PostAttachment capture.
|
||||
</span>
|
||||
<span v-else>
|
||||
An active batch is in progress. Wait for it to finish, or click
|
||||
<em>Clear stuck</em> above if it has been wedged with no
|
||||
measurable progress.
|
||||
</span>
|
||||
</p>
|
||||
|
||||
<div class="d-flex flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" rounded="pill"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'quick'"
|
||||
@click="trigger('quick')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-scan</v-icon>
|
||||
Quick scan
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="secondary" rounded="pill" variant="tonal"
|
||||
:disabled="!!store.activeBatch"
|
||||
:loading="busy === 'deep'"
|
||||
@click="trigger('deep')"
|
||||
>
|
||||
<v-icon start>mdi-magnify-plus-outline</v-icon>
|
||||
Deep scan
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert v-if="store.triggerError" type="error" variant="tonal" class="mt-3" closable>
|
||||
{{ store.triggerError }}
|
||||
</v-alert>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
|
||||
const store = useImportStore()
|
||||
const busy = ref(null)
|
||||
const clearing = ref(false)
|
||||
|
||||
async function trigger(mode) {
|
||||
busy.value = mode
|
||||
try { await store.triggerScan(mode) } catch {} finally { busy.value = null }
|
||||
}
|
||||
|
||||
async function onClearStuck() {
|
||||
clearing.value = true
|
||||
try {
|
||||
await store.clearStuck()
|
||||
} catch {
|
||||
// store surfaces error via triggerError if needed
|
||||
} finally {
|
||||
clearing.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -1,36 +1,59 @@
|
||||
<template>
|
||||
<div class="fc-maint">
|
||||
<p class="fc-muted text-body-2 mb-5">
|
||||
One-off backfills, tagging config and storage tools. Heads train nightly
|
||||
and auto-apply earned tags. Click a tile to open it.
|
||||
Processing, tagging and storage tools, grouped by system. Heads train
|
||||
nightly and auto-apply earned tags. Click a tile to open it.
|
||||
</p>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Backfills & reprocessing</h3>
|
||||
<p class="fc-section__hint">Re-run tagging, thumbnails, extraction and DB upkeep.</p>
|
||||
<div class="fc-tile-grid">
|
||||
<MLBackfillCard />
|
||||
<ThumbnailBackfillCard />
|
||||
<ArchiveReextractCard />
|
||||
<MissingFileRepairCard />
|
||||
<h3 class="fc-section__title">Ingestion & filters</h3>
|
||||
<p class="fc-section__hint">
|
||||
What gets imported — dedup sensitivity, size/transparency/solid-color
|
||||
filters. Applies to downloads and folder imports alike.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<ImportFiltersForm />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">GPU agent & embeddings</h3>
|
||||
<p class="fc-section__hint">
|
||||
The desktop agent that does the heavy lifting, its failure triage, and
|
||||
the CPU fallback.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<GpuAgentCard />
|
||||
<GpuTriageCard />
|
||||
<DbMaintenanceCard />
|
||||
<MLBackfillCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Tagging</h3>
|
||||
<p class="fc-section__hint">
|
||||
Suggestion thresholds, the auto-apply allowlist and tag aliases.
|
||||
Suggestion thresholds, trained heads and tag aliases.
|
||||
</p>
|
||||
<div class="fc-tile-stack">
|
||||
<MLThresholdSliders />
|
||||
<HeadsCard />
|
||||
<GpuAgentCard />
|
||||
<AliasTable />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Library health</h3>
|
||||
<p class="fc-section__hint">
|
||||
Self-healing and repair: missing files, thumbnails, database upkeep.
|
||||
</p>
|
||||
<div class="fc-tile-grid">
|
||||
<MissingFileRepairCard />
|
||||
<ThumbnailBackfillCard />
|
||||
<DbMaintenanceCard />
|
||||
<ArchiveReextractCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Storage</h3>
|
||||
<p class="fc-section__hint">Database + image backups and restore.</p>
|
||||
@@ -44,6 +67,7 @@
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted } from 'vue'
|
||||
|
||||
import ImportFiltersForm from './ImportFiltersForm.vue'
|
||||
import MLBackfillCard from './MLBackfillCard.vue'
|
||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
|
||||
@@ -18,14 +18,6 @@
|
||||
subtitle="Mark stranded pending/running events as error (also runs every 5 min)"
|
||||
@click="emit('recover-stalled')"
|
||||
/>
|
||||
<v-list-item
|
||||
:disabled="true"
|
||||
prepend-icon="mdi-download-box"
|
||||
title="Export failed logs"
|
||||
subtitle="CSV dump — v2"
|
||||
>
|
||||
<v-tooltip activator="parent" location="start">Deferred to a future release</v-tooltip>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</template>
|
||||
@@ -33,7 +25,8 @@
|
||||
<script setup>
|
||||
// The Downloads tab parent (DownloadsTab.vue) owns the actual retry/sweep
|
||||
// handlers — same toast + refresh logic already used by the failing-sources
|
||||
// RETRY ALL button. We just emit. Import-pipeline maintenance lives in
|
||||
// Settings → Imports (ImportTaskList.vue), not here.
|
||||
// RETRY ALL button. We just emit. (A permanently-disabled "Export failed
|
||||
// logs" stub sat here until 2026-07-02 — retired; the event list + detail
|
||||
// modal cover forensics.)
|
||||
const emit = defineEmits(['retry-failed', 'recover-stalled'])
|
||||
</script>
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
<template>
|
||||
<div>
|
||||
<!-- One extension home (2026-07-02): install/manifest card (moved here
|
||||
from Settings → Overview) + the API key bar it authenticates with.
|
||||
The extension feeds THIS view — cookies + one-click sources — so its
|
||||
setup lives with the rest of the ingestion config. -->
|
||||
<h3 class="text-h6 mb-3">Browser extension</h3>
|
||||
<BrowserExtensionCard class="mb-3" />
|
||||
<ExtensionKeyBar class="mb-4" />
|
||||
|
||||
<v-alert v-if="credentialsStore.error" type="error" variant="tonal" closable class="mb-4">
|
||||
{{ String(credentialsStore.error) }}
|
||||
</v-alert>
|
||||
|
||||
<h3 class="text-h6 mb-3">Platform credentials</h3>
|
||||
<h3 class="text-h6 mb-3 mt-6">Platform credentials</h3>
|
||||
<v-row>
|
||||
<v-col
|
||||
v-for="p in platformsStore.list"
|
||||
@@ -179,6 +185,7 @@ import { onMounted, reactive, ref, watch } from 'vue'
|
||||
import { usePlatformsStore } from '../../stores/platforms.js'
|
||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import BrowserExtensionCard from '../settings/BrowserExtensionCard.vue'
|
||||
import ExtensionKeyBar from '../credentials/ExtensionKeyBar.vue'
|
||||
import CredentialUploadDialog from '../credentials/CredentialUploadDialog.vue'
|
||||
import CredentialCard from './CredentialCard.vue'
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { toast } from '../utils/toast.js'
|
||||
import { ref, computed } from 'vue'
|
||||
import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
|
||||
// Import SETTINGS only. The manual-scan trigger + task-list surfaces retired
|
||||
// with the Import tab (2026-07-02): imports arrive via downloads/extension and
|
||||
// heal themselves (Layer-2 auto-refetch sweep); their runs show in Activity.
|
||||
// The /api/import/trigger + /tasks endpoints remain for direct/API use.
|
||||
// Consumers: ImportFiltersForm (Maintenance → Ingestion & filters) and the
|
||||
// Subscriptions Settings tab (downloader/extdl/schedule fields on the same
|
||||
// ImportSettings row).
|
||||
export const useImportStore = defineStore('import', () => {
|
||||
const api = useApi()
|
||||
|
||||
@@ -10,16 +17,6 @@ export const useImportStore = defineStore('import', () => {
|
||||
const settingsLoading = ref(false)
|
||||
const settingsError = ref(null)
|
||||
|
||||
const activeBatch = ref(null)
|
||||
const statusLoading = ref(false)
|
||||
|
||||
const tasks = ref([])
|
||||
const tasksNextCursor = ref(null)
|
||||
const tasksLoading = ref(false)
|
||||
const tasksFilter = ref({ status: null, limit: 50 })
|
||||
|
||||
const triggerError = ref(null)
|
||||
|
||||
async function loadSettings() {
|
||||
settingsLoading.value = true
|
||||
settingsError.value = null
|
||||
@@ -43,129 +40,8 @@ export const useImportStore = defineStore('import', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshStatus() {
|
||||
statusLoading.value = true
|
||||
try {
|
||||
const body = await api.get('/api/import/status')
|
||||
activeBatch.value = body.active_batch
|
||||
} finally {
|
||||
statusLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerScan(mode = 'quick') {
|
||||
if (!['quick', 'deep', 'verify'].includes(mode)) {
|
||||
throw new Error(`unsupported scan mode: ${mode}`)
|
||||
}
|
||||
triggerError.value = null
|
||||
try {
|
||||
await api.post('/api/import/trigger', { body: { mode } })
|
||||
// Acknowledge immediately so the click isn't invisible. scan_directory
|
||||
// can finalize the batch synchronously when every file in /import is
|
||||
// already on a non-failed ImportTask (operator-flagged 2026-05-25:
|
||||
// 233k existing tasks → all paths in skip-set → files_seen=0 →
|
||||
// batch flashes 'running' for <100ms then 'complete' before the
|
||||
// first refreshStatus() lands; UI never sees the active state).
|
||||
const label = mode === 'deep'
|
||||
? 'Deep scan triggered (re-applying sidecar metadata + filling NULL phash/artist on existing rows)'
|
||||
: mode === 'verify' ? 'Library verify triggered' : 'Quick scan triggered'
|
||||
toast({ text: label, type: 'success' })
|
||||
await refreshStatus()
|
||||
// Re-poll twice over ~5s and produce an HONEST follow-up toast.
|
||||
// Operator-flagged 2026-05-25: the prior "no new files" message was
|
||||
// misleading because deep scan IS doing work (refresh) even when
|
||||
// there are no new files to import. Surface the real workload count
|
||||
// (imported + refreshed + queued) instead. For quick scan + zero
|
||||
// queued work, fall back to "up to date" instead of the old
|
||||
// implementation-detail-leaking message.
|
||||
setTimeout(async () => {
|
||||
await refreshStatus()
|
||||
await loadTasks(true)
|
||||
if (activeBatch.value || mode === 'verify') return
|
||||
// Batch finalized quickly; figure out what actually happened.
|
||||
// The task list was just refreshed; the freshest row(s) carry
|
||||
// the batch outcome.
|
||||
const batchId = tasks.value[0]?.batch_id
|
||||
const sameBatch = batchId
|
||||
? tasks.value.filter(t => t.batch_id === batchId)
|
||||
: []
|
||||
const refreshedCount = sameBatch.filter(t => t.status === 'complete' && t.result_image_id).length
|
||||
const newImported = sameBatch.filter(t => t.status === 'complete' && t.result_image_id && !t.error).length
|
||||
if (mode === 'deep' && sameBatch.length > 0) {
|
||||
toast({
|
||||
text: `Deep scan finished — ${sameBatch.length} file(s) processed`,
|
||||
type: 'info',
|
||||
})
|
||||
} else if (mode === 'quick' && sameBatch.length > 0) {
|
||||
toast({
|
||||
text: `Quick scan finished — ${newImported} new file(s) queued`,
|
||||
type: 'info',
|
||||
})
|
||||
} else {
|
||||
toast({ text: 'Library is up to date', type: 'info' })
|
||||
}
|
||||
}, 2000)
|
||||
} catch (e) {
|
||||
triggerError.value = e.message
|
||||
toast({ text: `Scan failed: ${e.message}`, type: 'error' })
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function loadTasks(reset = true) {
|
||||
tasksLoading.value = true
|
||||
try {
|
||||
const params = { limit: tasksFilter.value.limit }
|
||||
if (tasksFilter.value.status) params.status = tasksFilter.value.status
|
||||
if (!reset && tasksNextCursor.value) params.cursor = tasksNextCursor.value
|
||||
const body = await api.get('/api/import/tasks', { params })
|
||||
tasks.value = reset ? body.tasks : [...tasks.value, ...body.tasks]
|
||||
tasksNextCursor.value = body.next_cursor
|
||||
} finally {
|
||||
tasksLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function setStatusFilter(status) {
|
||||
tasksFilter.value.status = status
|
||||
}
|
||||
|
||||
async function retryFailed() {
|
||||
await api.post('/api/import/retry-failed')
|
||||
await loadTasks(true)
|
||||
}
|
||||
|
||||
async function clearCompleted(ageDays = 0) {
|
||||
await api.post('/api/import/clear-completed', { body: { age_days: ageDays } })
|
||||
await loadTasks(true)
|
||||
}
|
||||
|
||||
async function clearStuck() {
|
||||
const body = await api.post('/api/import/clear-stuck')
|
||||
await loadTasks(true)
|
||||
await refreshStatus()
|
||||
return body
|
||||
}
|
||||
|
||||
// Layer-2 one-shot re-download for a failed task's (corrupt) file.
|
||||
// Returns the endpoint's status dict (refetch_queued / no_source /
|
||||
// already_refetched). Caller surfaces it as a toast.
|
||||
async function refetchTask(taskId) {
|
||||
const body = await api.post(`/api/import/tasks/${taskId}/refetch`)
|
||||
await loadTasks(true)
|
||||
return body
|
||||
}
|
||||
|
||||
const hasMore = computed(() => tasksNextCursor.value !== null)
|
||||
|
||||
return {
|
||||
settings, settingsLoading, settingsError,
|
||||
activeBatch, statusLoading,
|
||||
tasks, tasksLoading, tasksFilter, hasMore,
|
||||
triggerError,
|
||||
loadSettings, patchSettings,
|
||||
refreshStatus, triggerScan,
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck,
|
||||
refetchTask,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
<template>
|
||||
<v-container class="pt-3 pb-8">
|
||||
<h1 class="fc-h1 mb-4">{{ title }}</h1>
|
||||
<v-alert type="info" variant="tonal" icon="mdi-toolbox">
|
||||
This surface is a placeholder. It will be implemented in
|
||||
<strong>{{ surfaceOwner }}</strong>.
|
||||
</v-alert>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
|
||||
const route = useRoute()
|
||||
const title = computed(() => route.meta.title ?? 'FabledCurator')
|
||||
|
||||
const surfaceOwner = computed(() => {
|
||||
const fc2 = new Set(['gallery', 'showcase', 'tags', 'settings'])
|
||||
const fc3 = new Set(['subscriptions', 'credentials', 'downloads'])
|
||||
if (fc2.has(route.name)) return 'FC-2 (image backbone)'
|
||||
if (fc3.has(route.name)) return 'FC-3 (subscription backbone)'
|
||||
return 'a future sub-project'
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-h1 {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 32px;
|
||||
font-weight: 500;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
</style>
|
||||
@@ -17,7 +17,6 @@
|
||||
>
|
||||
<v-tab value="overview">Overview</v-tab>
|
||||
<v-tab value="activity">Activity</v-tab>
|
||||
<v-tab value="import">Import</v-tab>
|
||||
<v-tab value="cleanup">Cleanup</v-tab>
|
||||
<v-tab value="maintenance">Maintenance</v-tab>
|
||||
</v-tabs>
|
||||
@@ -34,31 +33,14 @@
|
||||
type="info" variant="tonal" class="mt-4" closable
|
||||
>
|
||||
{{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
|
||||
<v-btn variant="text" size="small" @click="tab = 'import'">Go to Import tab</v-btn>
|
||||
<v-btn variant="text" size="small" @click="tab = 'activity'">View in Activity</v-btn>
|
||||
</v-alert>
|
||||
<!-- Browser-extension install/download lives on Overview (moved
|
||||
from Maintenance 2026-05-25). Overview is the discovery
|
||||
surface for "things to set up"; Maintenance is for
|
||||
housekeeping of already-set-up systems. -->
|
||||
<BrowserExtensionCard class="mt-6" />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="activity">
|
||||
<SystemActivityTab />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="import">
|
||||
<!-- Order: filters → trigger → recent tasks. Filters hoisted above the
|
||||
trigger (operator-flagged 2026-06-04); the task list stays
|
||||
directly below the trigger so hit/miss feedback is adjacent to the
|
||||
button that produced it (operator-flagged 2026-05-25). -->
|
||||
<ImportFiltersForm />
|
||||
<v-divider class="my-6" />
|
||||
<ImportTriggerPanel />
|
||||
<v-divider class="my-6" />
|
||||
<ImportTaskList />
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="cleanup">
|
||||
<CleanupView />
|
||||
</v-window-item>
|
||||
@@ -73,21 +55,15 @@
|
||||
<script setup>
|
||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { useSystemStore } from '../stores/system.js'
|
||||
import { useImportStore } from '../stores/import.js'
|
||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||
import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
|
||||
import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
||||
import CleanupView from './CleanupView.vue'
|
||||
import { useMLStore } from '../stores/ml.js'
|
||||
|
||||
const tab = ref('overview')
|
||||
const system = useSystemStore()
|
||||
const importStore = useImportStore()
|
||||
const mlStore = useMLStore()
|
||||
|
||||
let pollId = null
|
||||
@@ -96,15 +72,7 @@ function startPolling() {
|
||||
if (pollId) return
|
||||
system.refreshStats()
|
||||
pollId = setInterval(() => {
|
||||
if (!document.hidden) {
|
||||
system.refreshStats()
|
||||
if (tab.value === 'import') {
|
||||
importStore.refreshStatus()
|
||||
// Refresh the task list while a batch is in flight so the UI
|
||||
// doesn't sit stale next to a ticking imported/skipped counter.
|
||||
if (importStore.activeBatch) importStore.loadTasks(true)
|
||||
}
|
||||
}
|
||||
if (!document.hidden) system.refreshStats()
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
@@ -112,21 +80,11 @@ function stopPolling() {
|
||||
if (pollId) { clearInterval(pollId); pollId = null }
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
startPolling()
|
||||
importStore.loadSettings()
|
||||
importStore.refreshStatus()
|
||||
importStore.loadTasks(true)
|
||||
})
|
||||
onMounted(startPolling)
|
||||
onUnmounted(stopPolling)
|
||||
|
||||
watch(tab, (t) => {
|
||||
if (t === 'import') {
|
||||
importStore.refreshStatus()
|
||||
importStore.loadTasks(true)
|
||||
} else if (t === 'maintenance') {
|
||||
mlStore.loadSettings()
|
||||
}
|
||||
if (t === 'maintenance') mlStore.loadSettings()
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
@@ -116,6 +116,30 @@ def test_transparent_filter(importer, import_layout):
|
||||
assert result.skip_reason == SkipReason.too_transparent
|
||||
|
||||
|
||||
def test_single_color_filter(importer, import_layout):
|
||||
"""The skip_single_color setting existed since FC-2 but was never wired
|
||||
(the audit module's docstring said so); wired 2026-07-02 using the same
|
||||
canonical predicate as the Cleanup audit. Solid fill skips when enabled,
|
||||
imports when disabled (the default)."""
|
||||
from PIL import Image as PILImage
|
||||
|
||||
import_root, _ = import_layout
|
||||
solid = import_root / "solid.png"
|
||||
solid.parent.mkdir(parents=True, exist_ok=True)
|
||||
PILImage.new("RGB", (100, 100), (12, 34, 56)).save(solid)
|
||||
|
||||
importer.settings.skip_single_color = True
|
||||
importer.settings.single_color_threshold = 0.95
|
||||
result = importer.import_one(solid)
|
||||
assert result.status == "skipped"
|
||||
assert result.skip_reason == SkipReason.single_color
|
||||
|
||||
importer.settings.skip_single_color = False
|
||||
solid2 = import_root / "solid2.png"
|
||||
PILImage.new("RGB", (100, 100), (200, 10, 10)).save(solid2)
|
||||
assert importer.import_one(solid2).status == "imported"
|
||||
|
||||
|
||||
def test_unsupported_extension(importer, import_layout):
|
||||
# FC-2d-iii: non-media is no longer skipped — it's captured as a
|
||||
# PostAttachment so nothing a post contained is lost.
|
||||
|
||||
Reference in New Issue
Block a user