From 5b34c9221c2101d41acfcdfa5c1c83ba52baccc0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 17:37:21 -0400 Subject: [PATCH] =?UTF-8?q?feat(ia):=20wave=201=20=E2=80=94=20Import=20tab?= =?UTF-8?q?=20dissolves,=20Maintenance=20regroups=20by=20system,=20one=20e?= =?UTF-8?q?xtension=20home?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings IA per the approved A3 design (the old layout was the two-app merge fossilized): - Import tab retired: ImportTriggerPanel + ImportTaskList deleted (manual /import scans stay API-level; imports arrive via downloads/extension, heal via the Layer-2 auto-refetch sweep, and show in Activity). ImportFiltersForm moves to Maintenance → 'Ingestion & filters' and loads its own settings; the import store shrinks to settings-only (no remaining consumers of the scan/task-list machinery). Overview's pending banner now points at Activity. - Maintenance regrouped: Ingestion & filters / GPU agent & embeddings (GpuAgent, Failed processing, CPU embedding backfill) / Tagging (sliders, Heads, Aliases) / Library health (MissingFiles, Thumbnails, DB, Archive re-extract demoted last) / Storage. - One extension home: BrowserExtensionCard moves from Settings → Overview to Subscriptions → Settings, above the API key bar it authenticates. - Single-color import filter WIRED: skip_single_color/threshold existed since FC-2 but nothing read them (the audit module's docstring said as much) — now enforced on both import paths via the audit's canonical predicate (tolerance 30, matching the Cleanup card default; animated images exempt like the transparency check). Default stays off; test added. - Dead weight: PlaceholderView (zero refs) and the permanently-disabled 'Export failed logs (CSV — v2)' menu stub deleted; stale docs fixed (celery queue docstring, threshold comment citing retired tasks, ml package docstring, HeadsCard 'replaces Camie' blurb). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/celery_app.py | 2 +- backend/app/services/audits/single_color.py | 9 +- backend/app/services/importer.py | 42 +++ backend/app/services/ml/__init__.py | 4 +- backend/app/tasks/maintenance.py | 7 +- .../src/components/settings/HeadsCard.vue | 2 +- .../components/settings/ImportFiltersForm.vue | 6 +- .../components/settings/ImportTaskList.vue | 254 ------------------ .../settings/ImportTriggerPanel.vue | 97 ------- .../components/settings/MaintenancePanel.vue | 48 +++- .../subscriptions/MaintenanceMenu.vue | 13 +- .../components/subscriptions/SettingsTab.vue | 9 +- frontend/src/stores/import.js | 140 +--------- frontend/src/views/PlaceholderView.vue | 34 --- frontend/src/views/SettingsView.vue | 50 +--- tests/test_importer.py | 24 ++ 16 files changed, 143 insertions(+), 598 deletions(-) delete mode 100644 frontend/src/components/settings/ImportTaskList.vue delete mode 100644 frontend/src/components/settings/ImportTriggerPanel.vue delete mode 100644 frontend/src/views/PlaceholderView.vue diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index f6d249e..74e2403 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -7,7 +7,7 @@ Queues: download — gallery-dl tasks (FC-3) scan — periodic source checks (FC-3) — kept separate so long imports don't starve the scheduler - maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3) + maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc. default — anything not explicitly routed """ diff --git a/backend/app/services/audits/single_color.py b/backend/app/services/audits/single_color.py index 167e078..6bddf07 100644 --- a/backend/app/services/audits/single_color.py +++ b/backend/app/services/audits/single_color.py @@ -1,8 +1,9 @@ """Single-color audit: matches images where one color dominates beyond -the threshold (within the given Euclidean RGB tolerance). The first -canonical implementation — the import-side filter (SkipReason.single_color) -was never wired; FC-Cleanup's audit module is the source of truth and a -future spec can adopt it on the import path too. +the threshold (within the given Euclidean RGB tolerance). The canonical +predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since +2026-07-02 — the import-side filter (Importer._single_color_hit / +SkipReason.single_color), so what the audit flags and what the import +skips can never disagree. """ from PIL import Image diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 8afd219..2459836 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -44,6 +44,7 @@ from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify from .archive_extractor import extract_archive, is_archive from .attachment_store import AttachmentStore +from .audits import single_color from .link_extract import extract_external_links from .thumbnailer import Thumbnailer @@ -790,6 +791,13 @@ class Importer: error=f"{pct:.2%} transparent", ) + if self.settings.skip_single_color and self._single_color_hit(source): + return ImportResult( + status="skipped", skip_reason=SkipReason.single_color, + error=(f"one color dominates >" + f"{self.settings.single_color_threshold:.0%}"), + ) + # Artist anchored to the attribution path (folder→artist), resolved # UP-FRONT so the enrich-on-duplicate branches link provenance with the # right artist even when the sidecar carries none — which is now the norm @@ -1123,6 +1131,13 @@ class Importer: status="skipped", skip_reason=SkipReason.too_transparent, error=f"{pct:.2%} transparent", ) + + if self.settings.skip_single_color and self._single_color_hit(path): + return ImportResult( + status="skipped", skip_reason=SkipReason.single_color, + error=(f"one color dominates >" + f"{self.settings.single_color_threshold:.0%}"), + ) else: # Best-effort probe for dims + duration so downloaded videos can dedup # (#871). LENIENT: unlike _import_media this path does not reject on a @@ -1538,6 +1553,33 @@ class Importer: # Benign orphan; the DB swap already committed. Don't undo it. pass + # Matches the Cleanup audit card's default tolerance: the import-side + # filter and the retroactive audit must agree on what "single color" MEANS + # (Euclidean RGB distance to the dominant color); only the match threshold + # is operator-tunable per surface. + _SINGLE_COLOR_TOLERANCE = 30 + + def _single_color_hit(self, source: Path) -> bool: + """True when one color dominates beyond the configured threshold — the + same canonical predicate the Cleanup audit runs (audits.single_color, + whose docstring anticipated this adoption; the skip_single_color + setting existed but was never wired until 2026-07-02). Never raises: + unreadable files were already rejected by verify() upstream, and a + residual decode error just declines to match (the import proceeds).""" + try: + with Image.open(source) as im: + if getattr(im, "is_animated", False): + # Frame 0 only would misjudge animations; skip like the + # transparency check does. + return False + return single_color.evaluate( + im, + threshold=self.settings.single_color_threshold, + tolerance=self._SINGLE_COLOR_TOLERANCE, + ) + except Exception: + return False + def _transparency_pct(self, source: Path) -> float: """Fraction of fully-transparent pixels in the image. 0.0 if no alpha. diff --git a/backend/app/services/ml/__init__.py b/backend/app/services/ml/__init__.py index 82ab712..723ddb4 100644 --- a/backend/app/services/ml/__init__.py +++ b/backend/app/services/ml/__init__.py @@ -1 +1,3 @@ -"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases.""" +"""ML pipeline services: embedders, heads (the learning suggester), suggestions, +GPU-job queue + failure triage, CCIP characters, crops/regions, allowlist and +aliases.""" diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 755cde8..f9549d9 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -139,10 +139,9 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = { "download": 30, # Audit 2026-06-02 — maintenance/scan queues run tasks that # legitimately exceed the 5-min default (verify_integrity at 70m - # hard, scan_directory at 70m hard, apply_allowlist_tags / - # recompute_centroids / backfill_phash at 35m hard). 75 min lives - # above the longest of those and the per-task overrides below - # cover the outliers (backups, library audit). + # hard, scan_directory at 70m hard, backfill_phash at 35m hard). + # 75 min lives above the longest of those and the per-task + # overrides below cover the outliers (backups, library audit). "maintenance": 75, "scan": 75, } diff --git a/frontend/src/components/settings/HeadsCard.vue b/frontend/src/components/settings/HeadsCard.vue index b9e684b..7117b0d 100644 --- a/frontend/src/components/settings/HeadsCard.vue +++ b/frontend/src/components/settings/HeadsCard.vue @@ -2,7 +2,7 @@

diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue index c538b35..3210d9e 100644 --- a/frontend/src/components/settings/ImportFiltersForm.vue +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -90,10 +90,14 @@ - - diff --git a/frontend/src/components/settings/ImportTriggerPanel.vue b/frontend/src/components/settings/ImportTriggerPanel.vue deleted file mode 100644 index 8873d92..0000000 --- a/frontend/src/components/settings/ImportTriggerPanel.vue +++ /dev/null @@ -1,97 +0,0 @@ - - - diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 69acbcc..26cae1f 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -1,36 +1,59 @@