bvandeusenandClaude Fable 5
7d48eb0b1b
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 7s
CI & Build / TypeScript typecheck (push) Successful in 12s
CI & Build / integration (push) Successful in 24s
CI & Build / Python tests (push) Successful in 55s
CI & Build / Build & push image (push) Successful in 24s
- background.start_periodic(interval, work, label=) replaces the three hand-rolled while-True/sleep/try loops in logging, auth and notifications. - services/scheduler.ScheduledJob replaces the four private BackgroundScheduler copies in recurrence/version_pinning/trash/db_maintenance schedulers; public start_/stop_/reschedule_ surfaces unchanged. - api_keys.hash_token is the one sha256 helper; auth.py used to inline it 5x. - auth.is_registration_open reads via settings.get_admin_setting; notification prefs read via settings.get_setting; _fire_share_email uses _get_user_email. - projects.get_project_summary / milestones.get_project_milestone_summary are now the one-id view of their batch siblings instead of a second copy of the queries; sharing.best_permission_by (was _deduplicate_by_permission) is the one rank-dedup, now also used by list_projects_for_user. - backup: the row builders for every section both exporters carry are named functions, so a column added to one export cannot silently miss the other. - iso() from models.base replaces the attr.isoformat()-if-attr-else-None idiom and db_maintenance._iso across services; backup keeps its explicit shape. - trash.py hoists the sql_delete/timedelta imports it re-imported per function. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1112 lines
47 KiB
Python
1112 lines
47 KiB
Python
import logging
|
|
from datetime import date, datetime, timezone
|
|
|
|
from sqlalchemy import or_, select
|
|
|
|
from scribe.models import async_session
|
|
from scribe.models.milestone import Milestone
|
|
from scribe.models.note import Note
|
|
from scribe.models.note_draft import NoteDraft
|
|
from scribe.models.note_supersession import NoteSupersession
|
|
from scribe.models.note_version import NoteVersion
|
|
from scribe.models.design_system import DesignSystem, DesignToken
|
|
from scribe.models.note_usage import NoteUsageEvent
|
|
from scribe.models.code_shape import CodeShape, CodeShapeEvent
|
|
from scribe.models.project import Project
|
|
from scribe.models.repo_binding import RepoBinding
|
|
from scribe.models.rulebook import (
|
|
Rule,
|
|
Rulebook,
|
|
RulebookTopic,
|
|
project_rule_suppressions,
|
|
project_rulebook_subscriptions,
|
|
project_topic_suppressions,
|
|
)
|
|
from scribe.models.setting import Setting
|
|
from scribe.models.system import RecordSystem, System
|
|
from scribe.models.task_log import TaskLog
|
|
from scribe.models.user import User
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Backup format version. v3 (2026-06) added rulebooks/topics/rules + their
|
|
# project subscription/suppression join tables. v4 (2026-07) dropped events
|
|
# when the calendar surface was retired — old v3 events are skipped on restore.
|
|
# v5 (2026-08) added the six tables that had accumulated outside the backup
|
|
# entirely (#2293), and the coverage guard that stops the seventh.
|
|
# v6 (2026-08) added note_supersessions — and the guard did stop the seventh:
|
|
# the table shipped without a backup section and the coverage test failed the
|
|
# build, which is the whole reason that list was written.
|
|
# v7 (2026-08) added code_shapes — the shape ledger (#2787). Classifications
|
|
# are judgment data worth carrying; a restore keeps a judgment only when its
|
|
# snippet target survives the id re-mapping, else the row rejoins the todo.
|
|
# v8 (2026-08) added code_shape_events — the ledger's history (#2793): what
|
|
# was used where, when, and why is not recomputable, so it travels.
|
|
# Bump when the serialized schema changes.
|
|
BACKUP_VERSION = 8
|
|
|
|
# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
|
|
# below, these two lists must together account for the entire schema — which is
|
|
# what tests/test_services_backup.py asserts against Base.metadata.
|
|
#
|
|
# The point is the ABSENCE case. A new table gets a model and a migration, both
|
|
# of which fail loudly if wrong, and then silently never gets a backup section:
|
|
# no error, no warning, and a restore that reports success. Naming the coverage
|
|
# explicitly turns "someone forgot" into a failing test (#2293).
|
|
_BACKED_UP = [
|
|
"users", "projects", "milestones", "notes", "task_logs", "note_drafts",
|
|
"note_versions", "settings", "rulebooks", "rulebook_topics", "rules",
|
|
"project_rulebook_subscriptions", "project_rule_suppressions",
|
|
"project_topic_suppressions",
|
|
# v5 (2026-08): the five-year gap this list was written to stop.
|
|
"systems", "record_systems", "design_systems", "design_tokens",
|
|
"note_usage_events", "repo_bindings", "note_supersessions",
|
|
# v7 (2026-08): the shape ledger (#2787); v8: its history (#2793).
|
|
"code_shapes", "code_shape_events",
|
|
]
|
|
|
|
# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
|
|
# explicit rather than silent. ACL (groups/shares) is a coherent follow-up;
|
|
# note_embeddings are derived (regenerated from note bodies); api_keys are
|
|
# sensitive credentials; retrieval_logs is observational telemetry that nothing
|
|
# reads for correctness and that grows per query; the rest are
|
|
# transient/operational.
|
|
#
|
|
# REAL table names, deliberately. This list used to read "embeddings",
|
|
# "invitations", "password_resets" — none of which are tables — so it looked
|
|
# like coverage while naming nothing the schema could confirm.
|
|
_NOT_INCLUDED = [
|
|
"groups", "group_memberships", "project_shares", "note_shares",
|
|
"api_keys", "note_embeddings", "app_logs", "notifications",
|
|
"invitation_tokens", "password_reset_tokens", "user_profiles",
|
|
"retrieval_logs",
|
|
# Sensitive credentials, same reasoning as api_keys: a backup that carries
|
|
# forge tokens is a token-exfiltration file. Users re-add connections
|
|
# after a restore; the per-project pin (projects.forge_connection_id) is
|
|
# deliberately not exported either, so restored projects fall back to
|
|
# keyring-by-host resolution — the documented unpinned behavior (#2778).
|
|
"forge_connections",
|
|
]
|
|
|
|
|
|
def _dt(val: str | None) -> datetime:
|
|
return datetime.fromisoformat(val) if val else datetime.now(timezone.utc)
|
|
|
|
|
|
def _d(val: str | None) -> date | None:
|
|
return date.fromisoformat(val) if val else None
|
|
|
|
|
|
def _subscription_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "rulebook_id": r.rulebook_id} for r in rows]
|
|
|
|
|
|
def _rule_suppression_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "rule_id": r.rule_id} for r in rows]
|
|
|
|
|
|
def _topic_suppression_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "topic_id": r.topic_id} for r in rows]
|
|
|
|
|
|
# The v5 sections. Pure row-builders like the join-table helpers above, for the
|
|
# same reason: CI has no database, so a serialiser that is a plain function is
|
|
# one that can actually be tested.
|
|
|
|
def _system_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "user_id": r.user_id, "project_id": r.project_id,
|
|
"name": r.name, "description": r.description, "color": r.color,
|
|
"status": r.status, "order_index": r.order_index,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _record_system_rows(rows) -> list[dict]:
|
|
return [{"note_id": r.note_id, "system_id": r.system_id} for r in rows]
|
|
|
|
|
|
def _note_supersession_rows(rows) -> list[dict]:
|
|
"""Which record has overtaken which. Carried because it is a JUDGEMENT —
|
|
someone decided this note replaced that one, and nothing in either note's
|
|
text records the decision. Lose it and the corpus silently reverts to
|
|
ranking stale material alongside current material."""
|
|
return [
|
|
{"superseder_id": r.superseder_id, "superseded_id": r.superseded_id}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _design_system_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "owner_user_id": r.owner_user_id, "title": r.title,
|
|
"description": r.description, "guidance": r.guidance,
|
|
"parent_id": r.parent_id,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _design_token_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "design_system_id": r.design_system_id, "name": r.name,
|
|
"value_by_mode": r.value_by_mode or {},
|
|
"group_name": r.group_name, "purpose": r.purpose,
|
|
"rationale": r.rationale, "supersedes": r.supersedes or [],
|
|
"order_index": r.order_index,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _usage_event_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"user_id": r.user_id, "note_id": r.note_id, "event": r.event,
|
|
"source": r.source,
|
|
"created_at": r.created_at.isoformat() if r.created_at else None,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _code_shape_rows(rows) -> list[dict]:
|
|
return [r.to_dict() for r in rows]
|
|
|
|
|
|
def _code_shape_event_rows(rows) -> list[dict]:
|
|
return [r.to_dict() for r in rows]
|
|
|
|
|
|
def _repo_binding_rows(rows) -> list[dict]:
|
|
return [
|
|
{"user_id": r.user_id, "project_id": r.project_id, "repo_key": r.repo_key}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
# Row builders for the sections both exporters carry. Pure, like the join-table
|
|
# helpers above; the full and per-user exports used to restate every one of
|
|
# these comprehensions side by side, and a column added to one and not the
|
|
# other is a backup that silently drops it (#2293's shape, one layer down).
|
|
|
|
def _user_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": u.id, "username": u.username, "email": u.email,
|
|
"password_hash": u.password_hash, "oauth_sub": u.oauth_sub,
|
|
"role": u.role, "session_version": u.session_version,
|
|
"created_at": u.created_at.isoformat(),
|
|
}
|
|
for u in rows
|
|
]
|
|
|
|
|
|
def _project_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": p.id, "user_id": p.user_id, "title": p.title,
|
|
"description": p.description, "goal": p.goal, "status": p.status,
|
|
"color": p.color,
|
|
"created_at": p.created_at.isoformat(),
|
|
"updated_at": p.updated_at.isoformat(),
|
|
}
|
|
for p in rows
|
|
]
|
|
|
|
|
|
def _milestone_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": m.id, "user_id": m.user_id, "project_id": m.project_id,
|
|
"title": m.title, "description": m.description, "status": m.status,
|
|
"order_index": m.order_index,
|
|
"created_at": m.created_at.isoformat(),
|
|
"updated_at": m.updated_at.isoformat(),
|
|
}
|
|
for m in rows
|
|
]
|
|
|
|
|
|
def _note_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": n.id, "user_id": n.user_id, "title": n.title, "body": n.body,
|
|
"tags": n.tags or [], "parent_id": n.parent_id,
|
|
"project_id": n.project_id, "milestone_id": n.milestone_id,
|
|
"status": n.status, "priority": n.priority,
|
|
"due_date": n.due_date.isoformat() if n.due_date else None,
|
|
"created_at": n.created_at.isoformat(),
|
|
"updated_at": n.updated_at.isoformat(),
|
|
}
|
|
for n in rows
|
|
]
|
|
|
|
|
|
def _task_log_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": tl.id, "user_id": tl.user_id, "task_id": tl.task_id,
|
|
"content": tl.content, "duration_minutes": tl.duration_minutes,
|
|
"created_at": tl.created_at.isoformat(),
|
|
"updated_at": tl.updated_at.isoformat(),
|
|
}
|
|
for tl in rows
|
|
]
|
|
|
|
|
|
def _note_draft_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": nd.id, "user_id": nd.user_id, "note_id": nd.note_id,
|
|
"proposed_body": nd.proposed_body, "original_body": nd.original_body,
|
|
"instruction": nd.instruction, "scope": nd.scope,
|
|
"created_at": nd.created_at.isoformat(),
|
|
"updated_at": nd.updated_at.isoformat(),
|
|
}
|
|
for nd in rows
|
|
]
|
|
|
|
|
|
def _note_version_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": nv.id, "user_id": nv.user_id, "note_id": nv.note_id,
|
|
"title": nv.title, "body": nv.body, "tags": nv.tags or [],
|
|
"pin_kind": nv.pin_kind, "pin_label": nv.pin_label,
|
|
"created_at": nv.created_at.isoformat(),
|
|
}
|
|
for nv in rows
|
|
]
|
|
|
|
|
|
def _setting_rows(rows) -> list[dict]:
|
|
return [{"user_id": s.user_id, "key": s.key, "value": s.value} for s in rows]
|
|
|
|
|
|
def _rulebook_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": rb.id, "owner_user_id": rb.owner_user_id, "title": rb.title,
|
|
"description": rb.description, "always_on": rb.always_on,
|
|
"created_at": rb.created_at.isoformat(),
|
|
"updated_at": rb.updated_at.isoformat(),
|
|
}
|
|
for rb in rows
|
|
]
|
|
|
|
|
|
def _topic_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": t.id, "rulebook_id": t.rulebook_id, "title": t.title,
|
|
"description": t.description, "order_index": t.order_index,
|
|
"created_at": t.created_at.isoformat(),
|
|
"updated_at": t.updated_at.isoformat(),
|
|
}
|
|
for t in rows
|
|
]
|
|
|
|
|
|
def _rule_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "topic_id": r.topic_id, "project_id": r.project_id,
|
|
"title": r.title, "statement": r.statement, "why": r.why,
|
|
"how_to_apply": r.how_to_apply, "order_index": r.order_index,
|
|
"created_at": r.created_at.isoformat(),
|
|
"updated_at": r.updated_at.isoformat(),
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Export
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def export_full_backup() -> dict:
|
|
"""Export all data as a version-5 JSON backup."""
|
|
async with async_session() as session:
|
|
users = (await session.execute(select(User))).scalars().all()
|
|
projects = (await session.execute(select(Project))).scalars().all()
|
|
milestones = (await session.execute(select(Milestone))).scalars().all()
|
|
notes = (await session.execute(select(Note))).scalars().all()
|
|
task_logs = (await session.execute(select(TaskLog))).scalars().all()
|
|
note_drafts = (await session.execute(select(NoteDraft))).scalars().all()
|
|
note_versions = (await session.execute(
|
|
select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id)
|
|
)).scalars().all()
|
|
settings = (await session.execute(select(Setting))).scalars().all()
|
|
systems = (await session.execute(select(System))).scalars().all()
|
|
record_systems = (await session.execute(select(RecordSystem))).scalars().all()
|
|
supersessions = (
|
|
await session.execute(select(NoteSupersession))
|
|
).scalars().all()
|
|
# Parent-first, so a restore can resolve parent_id as it goes rather
|
|
# than needing a second pass — the self-FK is the only ordering
|
|
# constraint in this payload.
|
|
design_systems = (await session.execute(
|
|
select(DesignSystem).order_by(DesignSystem.parent_id.nullsfirst(),
|
|
DesignSystem.id)
|
|
)).scalars().all()
|
|
design_tokens = (await session.execute(select(DesignToken))).scalars().all()
|
|
usage_events = (await session.execute(select(NoteUsageEvent))).scalars().all()
|
|
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
|
|
code_shapes = (await session.execute(select(CodeShape))).scalars().all()
|
|
code_shape_events = (await session.execute(
|
|
select(CodeShapeEvent).order_by(CodeShapeEvent.at, CodeShapeEvent.id)
|
|
)).scalars().all()
|
|
rulebooks = (await session.execute(select(Rulebook))).scalars().all()
|
|
topics = (await session.execute(select(RulebookTopic))).scalars().all()
|
|
rules = (await session.execute(select(Rule))).scalars().all()
|
|
subscriptions = (await session.execute(
|
|
select(project_rulebook_subscriptions)
|
|
)).all()
|
|
rule_suppressions = (await session.execute(
|
|
select(project_rule_suppressions)
|
|
)).all()
|
|
topic_suppressions = (await session.execute(
|
|
select(project_topic_suppressions)
|
|
)).all()
|
|
|
|
return {
|
|
"version": BACKUP_VERSION,
|
|
"scope": "full",
|
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
|
"_security_notice": (
|
|
"This backup contains hashed passwords. "
|
|
"Store it securely and restrict access."
|
|
),
|
|
"_not_included": _NOT_INCLUDED,
|
|
"users": _user_rows(users),
|
|
"projects": _project_rows(projects),
|
|
"milestones": _milestone_rows(milestones),
|
|
"notes": _note_rows(notes),
|
|
"task_logs": _task_log_rows(task_logs),
|
|
"note_drafts": _note_draft_rows(note_drafts),
|
|
"note_versions": _note_version_rows(note_versions),
|
|
"settings": _setting_rows(settings),
|
|
"rulebooks": _rulebook_rows(rulebooks),
|
|
"rulebook_topics": _topic_rows(topics),
|
|
"rules": _rule_rows(rules),
|
|
"rulebook_subscriptions": _subscription_rows(subscriptions),
|
|
"rule_suppressions": _rule_suppression_rows(rule_suppressions),
|
|
"topic_suppressions": _topic_suppression_rows(topic_suppressions),
|
|
"systems": _system_rows(systems),
|
|
"record_systems": _record_system_rows(record_systems),
|
|
"design_systems": _design_system_rows(design_systems),
|
|
"design_tokens": _design_token_rows(design_tokens),
|
|
"note_usage_events": _usage_event_rows(usage_events),
|
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
|
"note_supersessions": _note_supersession_rows(supersessions),
|
|
"code_shapes": _code_shape_rows(code_shapes),
|
|
"code_shape_events": _code_shape_event_rows(code_shape_events),
|
|
}
|
|
|
|
|
|
async def export_user_backup(user_id: int) -> dict:
|
|
"""Export a single user's data as a version-5 JSON backup."""
|
|
async with async_session() as session:
|
|
user = await session.get(User, user_id)
|
|
projects = (await session.execute(
|
|
select(Project).where(Project.user_id == user_id)
|
|
)).scalars().all()
|
|
project_ids = [p.id for p in projects]
|
|
milestones = (await session.execute(
|
|
select(Milestone).where(Milestone.user_id == user_id)
|
|
)).scalars().all()
|
|
notes = (await session.execute(
|
|
select(Note).where(Note.user_id == user_id)
|
|
)).scalars().all()
|
|
task_logs = (await session.execute(
|
|
select(TaskLog).where(TaskLog.user_id == user_id)
|
|
)).scalars().all()
|
|
note_drafts = (await session.execute(
|
|
select(NoteDraft).where(NoteDraft.user_id == user_id)
|
|
)).scalars().all()
|
|
note_versions = (await session.execute(
|
|
select(NoteVersion).where(NoteVersion.user_id == user_id)
|
|
.order_by(NoteVersion.note_id, NoteVersion.id)
|
|
)).scalars().all()
|
|
settings = (await session.execute(
|
|
select(Setting).where(Setting.user_id == user_id)
|
|
)).scalars().all()
|
|
systems = (await session.execute(
|
|
select(System).where(System.user_id == user_id)
|
|
)).scalars().all()
|
|
system_ids = [sy.id for sy in systems]
|
|
note_ids = [n.id for n in notes]
|
|
# Scoped by the user's SYSTEMS, not their notes: a shared note carrying
|
|
# this user's system tag belongs in their backup, and a note of theirs
|
|
# tagged with someone else's system does not — that row is the other
|
|
# user's to keep.
|
|
record_systems = (await session.execute(
|
|
select(RecordSystem).where(RecordSystem.system_id.in_(system_ids))
|
|
)).scalars().all() if system_ids else []
|
|
# BOTH ends must be this user's notes. A claim spanning out to someone
|
|
# else's record cannot be restored into a single-user import — the far
|
|
# id would not be in the map — so carrying it would export a row that
|
|
# silently vanishes on the way back in. Whole-instance backups have no
|
|
# such problem and take every row.
|
|
supersessions = (await session.execute(
|
|
select(NoteSupersession).where(
|
|
NoteSupersession.superseder_id.in_(note_ids),
|
|
NoteSupersession.superseded_id.in_(note_ids),
|
|
)
|
|
)).scalars().all() if note_ids else []
|
|
design_systems = (await session.execute(
|
|
select(DesignSystem).where(DesignSystem.owner_user_id == user_id)
|
|
.order_by(DesignSystem.parent_id.nullsfirst(), DesignSystem.id)
|
|
)).scalars().all()
|
|
ds_ids = [d.id for d in design_systems]
|
|
design_tokens = (await session.execute(
|
|
select(DesignToken).where(DesignToken.design_system_id.in_(ds_ids))
|
|
)).scalars().all() if ds_ids else []
|
|
usage_events = (await session.execute(
|
|
select(NoteUsageEvent).where(NoteUsageEvent.note_id.in_(note_ids))
|
|
)).scalars().all() if note_ids else []
|
|
repo_bindings = (await session.execute(
|
|
select(RepoBinding).where(RepoBinding.user_id == user_id)
|
|
)).scalars().all()
|
|
# The ledger has no user_id of its own — rows belong to the project
|
|
# they account for, so a user's export carries their projects' rows.
|
|
code_shapes = (await session.execute(
|
|
select(CodeShape).where(CodeShape.project_id.in_(project_ids))
|
|
)).scalars().all() if project_ids else []
|
|
code_shape_events = (await session.execute(
|
|
select(CodeShapeEvent).where(CodeShapeEvent.project_id.in_(project_ids))
|
|
.order_by(CodeShapeEvent.at, CodeShapeEvent.id)
|
|
)).scalars().all() if project_ids else []
|
|
rulebooks = (await session.execute(
|
|
select(Rulebook).where(Rulebook.owner_user_id == user_id)
|
|
)).scalars().all()
|
|
rulebook_ids = [rb.id for rb in rulebooks]
|
|
topics = (await session.execute(
|
|
select(RulebookTopic).where(RulebookTopic.rulebook_id.in_(rulebook_ids))
|
|
)).scalars().all() if rulebook_ids else []
|
|
topic_ids = [t.id for t in topics]
|
|
# Rules owned by the user = rules in the user's topics OR project-rules
|
|
# on the user's projects.
|
|
rule_filters = []
|
|
if topic_ids:
|
|
rule_filters.append(Rule.topic_id.in_(topic_ids))
|
|
if project_ids:
|
|
rule_filters.append(Rule.project_id.in_(project_ids))
|
|
rules = (await session.execute(
|
|
select(Rule).where(or_(*rule_filters))
|
|
)).scalars().all() if rule_filters else []
|
|
if project_ids:
|
|
subscriptions = (await session.execute(
|
|
select(project_rulebook_subscriptions).where(
|
|
project_rulebook_subscriptions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
rule_suppressions = (await session.execute(
|
|
select(project_rule_suppressions).where(
|
|
project_rule_suppressions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
topic_suppressions = (await session.execute(
|
|
select(project_topic_suppressions).where(
|
|
project_topic_suppressions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
else:
|
|
subscriptions = rule_suppressions = topic_suppressions = []
|
|
|
|
return {
|
|
"version": BACKUP_VERSION,
|
|
"scope": "user",
|
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
|
"_not_included": _NOT_INCLUDED,
|
|
"user": {
|
|
"id": user.id,
|
|
"username": user.username,
|
|
"email": user.email,
|
|
"role": user.role,
|
|
"created_at": user.created_at.isoformat(),
|
|
} if user else None,
|
|
"projects": _project_rows(projects),
|
|
"milestones": _milestone_rows(milestones),
|
|
"notes": _note_rows(notes),
|
|
"task_logs": _task_log_rows(task_logs),
|
|
"note_drafts": _note_draft_rows(note_drafts),
|
|
"note_versions": _note_version_rows(note_versions),
|
|
"settings": _setting_rows(settings),
|
|
"rulebooks": _rulebook_rows(rulebooks),
|
|
"rulebook_topics": _topic_rows(topics),
|
|
"rules": _rule_rows(rules),
|
|
"rulebook_subscriptions": _subscription_rows(subscriptions),
|
|
"rule_suppressions": _rule_suppression_rows(rule_suppressions),
|
|
"topic_suppressions": _topic_suppression_rows(topic_suppressions),
|
|
"systems": _system_rows(systems),
|
|
"record_systems": _record_system_rows(record_systems),
|
|
"design_systems": _design_system_rows(design_systems),
|
|
"design_tokens": _design_token_rows(design_tokens),
|
|
"note_usage_events": _usage_event_rows(usage_events),
|
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
|
"note_supersessions": _note_supersession_rows(supersessions),
|
|
"code_shapes": _code_shape_rows(code_shapes),
|
|
"code_shape_events": _code_shape_event_rows(code_shape_events),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Restore
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def restore_full_backup(data: dict) -> dict:
|
|
"""Restore from backup JSON. Dispatches by version."""
|
|
version = data.get("version", 1)
|
|
if version == 1:
|
|
return await _restore_v1(data)
|
|
# v2 and v3 share one path; v3-only sections are guarded by data.get so a
|
|
# v2 payload (without them) restores cleanly.
|
|
return await _restore_v2(data)
|
|
|
|
|
|
async def _restore_v1(data: dict) -> dict:
|
|
"""Restore legacy v1 backup (original format).
|
|
|
|
Pre-pivot v1 backups included conversations + messages; those are
|
|
skipped during restore now that the chat subsystem is gone.
|
|
"""
|
|
stats = {"users": 0, "notes": 0, "settings": 0}
|
|
|
|
async with async_session() as session:
|
|
user_id_map: dict[int, int] = {}
|
|
for u_data in data.get("users", []):
|
|
old_id = u_data["id"]
|
|
user = User(
|
|
username=u_data["username"],
|
|
email=u_data.get("email"),
|
|
password_hash=u_data["password_hash"],
|
|
role=u_data.get("role", "user"),
|
|
created_at=_dt(u_data.get("created_at")),
|
|
)
|
|
session.add(user)
|
|
await session.flush()
|
|
user_id_map[old_id] = user.id
|
|
stats["users"] += 1
|
|
|
|
note_id_map: dict[int, int] = {}
|
|
for n_data in data.get("notes", []):
|
|
old_id = n_data.get("id")
|
|
mapped_user_id = user_id_map.get(n_data.get("user_id", 0))
|
|
if mapped_user_id is None:
|
|
continue
|
|
note = Note(
|
|
user_id=mapped_user_id,
|
|
title=n_data.get("title", ""),
|
|
body=n_data.get("body", ""),
|
|
tags=n_data.get("tags", []),
|
|
parent_id=None, # patched below
|
|
status=n_data.get("status"),
|
|
priority=n_data.get("priority"),
|
|
due_date=_d(n_data.get("due_date")),
|
|
created_at=_dt(n_data.get("created_at")),
|
|
updated_at=_dt(n_data.get("updated_at")),
|
|
)
|
|
session.add(note)
|
|
await session.flush()
|
|
if old_id is not None:
|
|
note_id_map[old_id] = note.id
|
|
stats["notes"] += 1
|
|
|
|
# Patch parent_id now that all notes have new IDs
|
|
for n_data in data.get("notes", []):
|
|
old_id = n_data.get("id")
|
|
old_parent = n_data.get("parent_id")
|
|
if old_id and old_parent and old_id in note_id_map and old_parent in note_id_map:
|
|
note_row = await session.get(Note, note_id_map[old_id])
|
|
if note_row:
|
|
note_row.parent_id = note_id_map[old_parent]
|
|
|
|
for s_data in data.get("settings", []):
|
|
mapped_user_id = user_id_map.get(s_data.get("user_id", 0))
|
|
if mapped_user_id is None:
|
|
continue
|
|
session.add(Setting(user_id=mapped_user_id, key=s_data["key"], value=s_data.get("value", "")))
|
|
stats["settings"] += 1
|
|
|
|
await session.commit()
|
|
|
|
logger.info("Restored v1 backup: %s", stats)
|
|
return stats
|
|
|
|
|
|
async def _restore_v2(data: dict) -> dict:
|
|
"""Restore v2/v3 backup with full FK re-mapping.
|
|
|
|
Conversations, push subscriptions, and (as of v4) events in older backups
|
|
are silently skipped — those subsystems were removed. v3+ sections
|
|
(rulebooks/topics/rules/join-tables) are guarded by data.get so a v2
|
|
payload restores without them.
|
|
"""
|
|
stats: dict[str, int] = {
|
|
"users": 0, "projects": 0, "milestones": 0, "notes": 0,
|
|
"task_logs": 0, "note_drafts": 0, "note_versions": 0,
|
|
"settings": 0, "rulebooks": 0, "rulebook_topics": 0, "rules": 0,
|
|
"rulebook_subscriptions": 0, "rule_suppressions": 0,
|
|
"topic_suppressions": 0,
|
|
"systems": 0, "record_systems": 0, "design_systems": 0,
|
|
"design_tokens": 0, "note_usage_events": 0, "repo_bindings": 0,
|
|
"note_supersessions": 0, "code_shapes": 0, "code_shape_events": 0,
|
|
}
|
|
|
|
async with async_session() as session:
|
|
user_id_map: dict[int, int] = {}
|
|
project_id_map: dict[int, int] = {}
|
|
milestone_id_map: dict[int, int] = {}
|
|
note_id_map: dict[int, int] = {}
|
|
rulebook_id_map: dict[int, int] = {}
|
|
topic_id_map: dict[int, int] = {}
|
|
rule_id_map: dict[int, int] = {}
|
|
|
|
# 1. Users
|
|
for u_data in data.get("users", []):
|
|
old_id = u_data["id"]
|
|
user = User(
|
|
username=u_data["username"],
|
|
email=u_data.get("email"),
|
|
password_hash=u_data.get("password_hash"),
|
|
oauth_sub=u_data.get("oauth_sub"),
|
|
role=u_data.get("role", "user"),
|
|
session_version=u_data.get("session_version", 1),
|
|
created_at=_dt(u_data.get("created_at")),
|
|
)
|
|
session.add(user)
|
|
await session.flush()
|
|
user_id_map[old_id] = user.id
|
|
stats["users"] += 1
|
|
|
|
# 2. Projects
|
|
for p_data in data.get("projects", []):
|
|
mapped_uid = user_id_map.get(p_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
proj = Project(
|
|
user_id=mapped_uid,
|
|
title=p_data.get("title", ""),
|
|
description=p_data.get("description", ""),
|
|
goal=p_data.get("goal", ""),
|
|
status=p_data.get("status", "active"),
|
|
color=p_data.get("color"),
|
|
created_at=_dt(p_data.get("created_at")),
|
|
updated_at=_dt(p_data.get("updated_at")),
|
|
)
|
|
session.add(proj)
|
|
await session.flush()
|
|
project_id_map[p_data["id"]] = proj.id
|
|
stats["projects"] += 1
|
|
|
|
# 3. Milestones
|
|
for m_data in data.get("milestones", []):
|
|
mapped_uid = user_id_map.get(m_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(m_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
ms = Milestone(
|
|
user_id=mapped_uid,
|
|
project_id=mapped_pid,
|
|
title=m_data.get("title", ""),
|
|
description=m_data.get("description"),
|
|
status=m_data.get("status", "active"),
|
|
order_index=m_data.get("order_index", 0),
|
|
created_at=_dt(m_data.get("created_at")),
|
|
updated_at=_dt(m_data.get("updated_at")),
|
|
)
|
|
session.add(ms)
|
|
await session.flush()
|
|
milestone_id_map[m_data["id"]] = ms.id
|
|
stats["milestones"] += 1
|
|
|
|
# 4a. Notes — first pass (no parent_id yet)
|
|
notes_with_parents: list[tuple[int, int]] = [] # (new_note_id, old_parent_id)
|
|
for n_data in data.get("notes", []):
|
|
mapped_uid = user_id_map.get(n_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
note = Note(
|
|
user_id=mapped_uid,
|
|
title=n_data.get("title", ""),
|
|
body=n_data.get("body", ""),
|
|
tags=n_data.get("tags", []),
|
|
parent_id=None,
|
|
project_id=project_id_map.get(n_data["project_id"]) if n_data.get("project_id") else None,
|
|
milestone_id=milestone_id_map.get(n_data["milestone_id"]) if n_data.get("milestone_id") else None,
|
|
status=n_data.get("status"),
|
|
priority=n_data.get("priority"),
|
|
due_date=_d(n_data.get("due_date")),
|
|
created_at=_dt(n_data.get("created_at")),
|
|
updated_at=_dt(n_data.get("updated_at")),
|
|
)
|
|
session.add(note)
|
|
await session.flush()
|
|
note_id_map[n_data["id"]] = note.id
|
|
if n_data.get("parent_id"):
|
|
notes_with_parents.append((note.id, n_data["parent_id"]))
|
|
stats["notes"] += 1
|
|
|
|
# 4b. Patch parent_id
|
|
for new_note_id, old_parent_id in notes_with_parents:
|
|
new_parent_id = note_id_map.get(old_parent_id)
|
|
if new_parent_id:
|
|
note_row = await session.get(Note, new_note_id)
|
|
if note_row:
|
|
note_row.parent_id = new_parent_id
|
|
|
|
# 5. TaskLogs
|
|
for tl_data in data.get("task_logs", []):
|
|
mapped_uid = user_id_map.get(tl_data.get("user_id", 0))
|
|
mapped_tid = note_id_map.get(tl_data.get("task_id", 0))
|
|
if mapped_uid is None or mapped_tid is None:
|
|
continue
|
|
tl = TaskLog(
|
|
user_id=mapped_uid,
|
|
task_id=mapped_tid,
|
|
content=tl_data.get("content", ""),
|
|
duration_minutes=tl_data.get("duration_minutes"),
|
|
created_at=_dt(tl_data.get("created_at")),
|
|
updated_at=_dt(tl_data.get("updated_at")),
|
|
)
|
|
session.add(tl)
|
|
stats["task_logs"] += 1
|
|
|
|
# 6. NoteDrafts
|
|
for nd_data in data.get("note_drafts", []):
|
|
mapped_uid = user_id_map.get(nd_data.get("user_id", 0))
|
|
mapped_nid = note_id_map.get(nd_data.get("note_id", 0))
|
|
if mapped_uid is None or mapped_nid is None:
|
|
continue
|
|
nd = NoteDraft(
|
|
user_id=mapped_uid,
|
|
note_id=mapped_nid,
|
|
proposed_body=nd_data.get("proposed_body", ""),
|
|
original_body=nd_data.get("original_body", ""),
|
|
instruction=nd_data.get("instruction", ""),
|
|
scope=nd_data.get("scope", "document"),
|
|
created_at=_dt(nd_data.get("created_at")),
|
|
updated_at=_dt(nd_data.get("updated_at")),
|
|
)
|
|
session.add(nd)
|
|
stats["note_drafts"] += 1
|
|
|
|
# 7. NoteVersions
|
|
for nv_data in data.get("note_versions", []):
|
|
mapped_uid = user_id_map.get(nv_data.get("user_id", 0))
|
|
mapped_nid = note_id_map.get(nv_data.get("note_id", 0))
|
|
if mapped_uid is None or mapped_nid is None:
|
|
continue
|
|
nv = NoteVersion(
|
|
user_id=mapped_uid,
|
|
note_id=mapped_nid,
|
|
title=nv_data.get("title", ""),
|
|
body=nv_data.get("body", ""),
|
|
tags=nv_data.get("tags", []),
|
|
pin_kind=nv_data.get("pin_kind"),
|
|
pin_label=nv_data.get("pin_label"),
|
|
created_at=_dt(nv_data.get("created_at")),
|
|
)
|
|
session.add(nv)
|
|
stats["note_versions"] += 1
|
|
|
|
# 8. Settings
|
|
for s_data in data.get("settings", []):
|
|
mapped_uid = user_id_map.get(s_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
|
|
stats["settings"] += 1
|
|
|
|
# 9. Rulebooks (v3)
|
|
for rb_data in data.get("rulebooks", []):
|
|
mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
rb = Rulebook(
|
|
owner_user_id=mapped_uid,
|
|
title=rb_data.get("title", ""),
|
|
description=rb_data.get("description", ""),
|
|
always_on=rb_data.get("always_on", False),
|
|
created_at=_dt(rb_data.get("created_at")),
|
|
updated_at=_dt(rb_data.get("updated_at")),
|
|
)
|
|
session.add(rb)
|
|
await session.flush()
|
|
rulebook_id_map[rb_data["id"]] = rb.id
|
|
stats["rulebooks"] += 1
|
|
|
|
# 10. Topics (v3)
|
|
for t_data in data.get("rulebook_topics", []):
|
|
mapped_rbid = rulebook_id_map.get(t_data.get("rulebook_id", 0))
|
|
if mapped_rbid is None:
|
|
continue
|
|
topic = RulebookTopic(
|
|
rulebook_id=mapped_rbid,
|
|
title=t_data.get("title", ""),
|
|
description=t_data.get("description"),
|
|
order_index=t_data.get("order_index", 0),
|
|
created_at=_dt(t_data.get("created_at")),
|
|
updated_at=_dt(t_data.get("updated_at")),
|
|
)
|
|
session.add(topic)
|
|
await session.flush()
|
|
topic_id_map[t_data["id"]] = topic.id
|
|
stats["rulebook_topics"] += 1
|
|
|
|
# 11. Rules (v3) — topic-rule (topic_id) XOR project-rule (project_id)
|
|
for r_data in data.get("rules", []):
|
|
mapped_topic = topic_id_map.get(r_data["topic_id"]) if r_data.get("topic_id") else None
|
|
mapped_proj = project_id_map.get(r_data["project_id"]) if r_data.get("project_id") else None
|
|
if mapped_topic is None and mapped_proj is None:
|
|
continue # orphaned — its parent didn't restore
|
|
rule = Rule(
|
|
topic_id=mapped_topic,
|
|
project_id=mapped_proj,
|
|
title=r_data.get("title", ""),
|
|
statement=r_data.get("statement", ""),
|
|
why=r_data.get("why") or None,
|
|
how_to_apply=r_data.get("how_to_apply") or None,
|
|
order_index=r_data.get("order_index", 0),
|
|
created_at=_dt(r_data.get("created_at")),
|
|
updated_at=_dt(r_data.get("updated_at")),
|
|
)
|
|
session.add(rule)
|
|
await session.flush()
|
|
rule_id_map[r_data["id"]] = rule.id
|
|
stats["rules"] += 1
|
|
|
|
# 12. Rulebook subscriptions (v3 join table)
|
|
for sub in data.get("rulebook_subscriptions", []):
|
|
mapped_pid = project_id_map.get(sub.get("project_id", 0))
|
|
mapped_rbid = rulebook_id_map.get(sub.get("rulebook_id", 0))
|
|
if mapped_pid is None or mapped_rbid is None:
|
|
continue
|
|
await session.execute(project_rulebook_subscriptions.insert().values(
|
|
project_id=mapped_pid, rulebook_id=mapped_rbid,
|
|
))
|
|
stats["rulebook_subscriptions"] += 1
|
|
|
|
# 13. Rule suppressions (v3 join table)
|
|
for sup in data.get("rule_suppressions", []):
|
|
mapped_pid = project_id_map.get(sup.get("project_id", 0))
|
|
mapped_rid = rule_id_map.get(sup.get("rule_id", 0))
|
|
if mapped_pid is None or mapped_rid is None:
|
|
continue
|
|
await session.execute(project_rule_suppressions.insert().values(
|
|
project_id=mapped_pid, rule_id=mapped_rid,
|
|
))
|
|
stats["rule_suppressions"] += 1
|
|
|
|
# 14. Topic suppressions (v3 join table)
|
|
for sup in data.get("topic_suppressions", []):
|
|
mapped_pid = project_id_map.get(sup.get("project_id", 0))
|
|
mapped_tid = topic_id_map.get(sup.get("topic_id", 0))
|
|
if mapped_pid is None or mapped_tid is None:
|
|
continue
|
|
await session.execute(project_topic_suppressions.insert().values(
|
|
project_id=mapped_pid, topic_id=mapped_tid,
|
|
))
|
|
stats["topic_suppressions"] += 1
|
|
|
|
# --- v5 sections. Every one is data.get()-guarded, so a v2/v3/v4
|
|
# payload restores without them rather than failing on an absent key.
|
|
|
|
# 15. Systems
|
|
system_id_map: dict[int, int] = {}
|
|
for sy_data in data.get("systems", []):
|
|
mapped_uid = user_id_map.get(sy_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(sy_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
system = System(
|
|
user_id=mapped_uid, project_id=mapped_pid,
|
|
name=sy_data.get("name", ""),
|
|
description=sy_data.get("description"),
|
|
color=sy_data.get("color"),
|
|
status=sy_data.get("status", "active"),
|
|
order_index=sy_data.get("order_index", 0),
|
|
)
|
|
session.add(system)
|
|
await session.flush()
|
|
system_id_map[sy_data["id"]] = system.id
|
|
stats["systems"] += 1
|
|
|
|
# 16. Record↔system links
|
|
for rs in data.get("record_systems", []):
|
|
mapped_nid = note_id_map.get(rs.get("note_id", 0))
|
|
mapped_sid = system_id_map.get(rs.get("system_id", 0))
|
|
if mapped_nid is None or mapped_sid is None:
|
|
continue
|
|
session.add(RecordSystem(note_id=mapped_nid, system_id=mapped_sid))
|
|
stats["record_systems"] += 1
|
|
|
|
# 16b. Supersession claims. Guarded by `data.get` like every other
|
|
# post-v2 section, so a v5 or older payload restores cleanly without it.
|
|
#
|
|
# Both ends must map. A claim is about a PAIR — half of one is not a
|
|
# weaker claim, it is a dangling row pointing at whatever note happens
|
|
# to hold that id next.
|
|
for sup in data.get("note_supersessions", []):
|
|
mapped_new = note_id_map.get(sup.get("superseder_id", 0))
|
|
mapped_old = note_id_map.get(sup.get("superseded_id", 0))
|
|
if mapped_new is None or mapped_old is None or mapped_new == mapped_old:
|
|
continue
|
|
session.add(
|
|
NoteSupersession(
|
|
superseder_id=mapped_new, superseded_id=mapped_old
|
|
)
|
|
)
|
|
stats["note_supersessions"] += 1
|
|
|
|
# 17. Design systems. The export orders these parent-first, so a
|
|
# parent's new id is always in the map by the time a child needs it —
|
|
# no second pass, and a child whose parent is missing lands as a root
|
|
# rather than failing the whole restore.
|
|
design_system_id_map: dict[int, int] = {}
|
|
for ds_data in data.get("design_systems", []):
|
|
mapped_uid = user_id_map.get(ds_data.get("owner_user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
design = DesignSystem(
|
|
owner_user_id=mapped_uid,
|
|
title=ds_data.get("title", ""),
|
|
description=ds_data.get("description"),
|
|
guidance=ds_data.get("guidance"),
|
|
parent_id=design_system_id_map.get(ds_data.get("parent_id") or 0),
|
|
)
|
|
session.add(design)
|
|
await session.flush()
|
|
design_system_id_map[ds_data["id"]] = design.id
|
|
stats["design_systems"] += 1
|
|
|
|
# 18. Design tokens
|
|
for t_data in data.get("design_tokens", []):
|
|
mapped_dsid = design_system_id_map.get(t_data.get("design_system_id", 0))
|
|
if mapped_dsid is None:
|
|
continue
|
|
session.add(DesignToken(
|
|
design_system_id=mapped_dsid,
|
|
name=t_data.get("name", ""),
|
|
value_by_mode=t_data.get("value_by_mode") or {},
|
|
group_name=t_data.get("group_name"),
|
|
purpose=t_data.get("purpose"),
|
|
rationale=t_data.get("rationale"),
|
|
supersedes=t_data.get("supersedes") or [],
|
|
order_index=t_data.get("order_index", 0),
|
|
))
|
|
stats["design_tokens"] += 1
|
|
|
|
# 19. Usage events. Kept because pull-through is the evidence base for
|
|
# whether recall works at all, and it is only ever accumulated — a
|
|
# restore that dropped it would silently reset that measurement to zero
|
|
# while everything still looked fine.
|
|
for ev in data.get("note_usage_events", []):
|
|
mapped_nid = note_id_map.get(ev.get("note_id", 0))
|
|
if mapped_nid is None:
|
|
continue
|
|
session.add(NoteUsageEvent(
|
|
user_id=user_id_map.get(ev.get("user_id") or 0),
|
|
note_id=mapped_nid,
|
|
event=ev.get("event", ""),
|
|
source=ev.get("source", ""),
|
|
created_at=_dt(ev.get("created_at")),
|
|
))
|
|
stats["note_usage_events"] += 1
|
|
|
|
# 20. Repo bindings — small, but losing them means every bound repo
|
|
# quietly stops loading its project at session start.
|
|
for rb_data in data.get("repo_bindings", []):
|
|
mapped_uid = user_id_map.get(rb_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(rb_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
session.add(RepoBinding(
|
|
user_id=mapped_uid, project_id=mapped_pid,
|
|
repo_key=rb_data.get("repo_key", ""),
|
|
))
|
|
stats["repo_bindings"] += 1
|
|
|
|
# 21. Code shapes (v7, #2787) — the ledger's classifications are
|
|
# judgments worth carrying. A judgment whose snippet target didn't
|
|
# survive the re-mapping (canonical/instance/variant with a gone
|
|
# snippet) is downgraded to unclassified so it rejoins the todo
|
|
# honestly instead of dangling; exempt needs no target and keeps.
|
|
shape_id_map: dict[int, int] = {}
|
|
for cs_data in data.get("code_shapes", []):
|
|
mapped_pid = project_id_map.get(cs_data.get("project_id", 0))
|
|
if mapped_pid is None:
|
|
continue
|
|
status = cs_data.get("status", "unclassified")
|
|
mapped_sid = note_id_map.get(cs_data.get("snippet_id") or 0)
|
|
classified_by = cs_data.get("classified_by")
|
|
classified_at = cs_data.get("classified_at")
|
|
if status in ("canonical", "instance", "variant") and mapped_sid is None:
|
|
status = "unclassified"
|
|
classified_by = None
|
|
classified_at = None
|
|
shape = CodeShape(
|
|
project_id=mapped_pid,
|
|
repo_key=cs_data.get("repo_key", ""),
|
|
path=cs_data.get("path", ""),
|
|
symbol=cs_data.get("symbol", ""),
|
|
kind=cs_data.get("kind", "sym"),
|
|
status=status,
|
|
snippet_id=mapped_sid,
|
|
reason=cs_data.get("reason"),
|
|
classified_by=classified_by,
|
|
classified_at=_dt(classified_at) if classified_at else None,
|
|
first_seen_commit=cs_data.get("first_seen_commit", ""),
|
|
last_seen_commit=cs_data.get("last_seen_commit", ""),
|
|
vanished_at=_dt(cs_data["vanished_at"]) if cs_data.get("vanished_at") else None,
|
|
# Fingerprints restore; proposals (#2792) deliberately do not —
|
|
# they are mechanical, and the next refresh recomputes them
|
|
# against the restored snippet ids.
|
|
signature=cs_data.get("signature", ""),
|
|
body_sha=cs_data.get("body_sha", ""),
|
|
classified_sha=cs_data.get("classified_sha", ""),
|
|
created_at=_dt(cs_data.get("created_at")),
|
|
updated_at=_dt(cs_data.get("updated_at")),
|
|
)
|
|
session.add(shape)
|
|
await session.flush()
|
|
if cs_data.get("id"):
|
|
shape_id_map[int(cs_data["id"])] = shape.id
|
|
stats["code_shapes"] += 1
|
|
|
|
# v8: the ledger's history rides its shapes. snippet_id is kept as
|
|
# the history's own claim (FK-free by design) but re-mapped when the
|
|
# snippet survived, so a restored timeline points at restored records.
|
|
for ev in data.get("code_shape_events", []):
|
|
new_shape_id = shape_id_map.get(ev.get("shape_id") or 0)
|
|
mapped_pid = project_id_map.get(ev.get("project_id", 0))
|
|
if new_shape_id is None or mapped_pid is None:
|
|
continue
|
|
old_sid = ev.get("snippet_id")
|
|
session.add(CodeShapeEvent(
|
|
shape_id=new_shape_id,
|
|
project_id=mapped_pid,
|
|
path=ev.get("path", ""),
|
|
symbol=ev.get("symbol", ""),
|
|
kind=ev.get("kind", "sym"),
|
|
event=ev.get("event", "classified"),
|
|
status=ev.get("status"),
|
|
snippet_id=note_id_map.get(old_sid, old_sid) if old_sid else None,
|
|
classified_by=ev.get("classified_by"),
|
|
reason=ev.get("reason"),
|
|
commit=ev.get("commit", ""),
|
|
at=_dt(ev.get("at")),
|
|
))
|
|
stats["code_shape_events"] += 1
|
|
|
|
await session.commit()
|
|
|
|
logger.info("Restored v2/v3 backup: %s", stats)
|
|
return stats
|