attachments sync: attach offline, upload when linked, removals stick (#5168)

Desktop could not create an attachment at all, and a removed attachment or
dismissed preview came back on the next pull. Now:

- core: add_attachment keeps the bytes in the blob store and queues the row
  (schema v10: attachments.uploaded / upload_error). Push uploads it once its
  note has landed. A refusal that retrying won't fix (too large, id clash, hash
  mismatch) is recorded on the file and not re-sent every cycle; the editor
  shows it.
- core: removing a synced attachment or dismissing a preview leaves a tombstone
  in pending_deletes; push sends it as an `attachment`/`preview` delete, and a
  pull while it waits doesn't put the row back. A pull also keeps files still
  waiting to upload instead of replacing them wholesale.
- server: PUT /api/sync/attachments/<id> (raw body, sha256-checked, idempotent,
  size-capped) and child deletes in push, which apply regardless of LWW and
  answer noop for rows the caller can't see. One store_attachment helper for
  the upload route, the importer and sync. Protocol 5, feature attachment_sync;
  the client sends neither to a server without it.
- server: migration 0031 makes a link preview's insert/delete bump its note, so
  background-fetched previews and web dismissals reach linked devices.
- desktop: Attach and paste-image work offline (raw-bytes IPC command).
- SVG is served as a download by the desktop blob scheme too (as #1981 did for
  the web), and drawn as a file chip on both.
- autosync: drop the catch_unwind; release builds abort on panic, so it only
  ever worked in debug builds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/src/sync/push.rs

@@ -1,8 +1,9 @@
 //! Push: send local changes to the server and apply what it says (M10.7c).
 //!
-//! Two sources feed a push: rows flagged `dirty` (created or edited locally) and rows
+//! Three sources feed a push: rows flagged `dirty` (created or edited locally), rows
 //! in `pending_deletes` (permanently deleted locally — see `local::schema` v2 for why
-//! a delete needs its own record).
+//! a delete needs its own record), and files attached on this device that are still
+//! waiting to go up (`attachments.uploaded = 0`, schema v10).
 //!
 //! Sync is **whole-note**: an upsert carries the client's full current state, not a
 //! patch (docs/sync.md). The server resolves conflicts last-write-wins by the client's
@@ -11,7 +12,8 @@
 use rusqlite::{params, Connection, OptionalExtension};
 use serde::{Deserialize, Serialize};
 
-use super::client;
+use super::blobs::BlobStore;
+use super::client::{self, UploadError};
 use super::state;
 use crate::local::Db;
 
@@ -35,6 +37,11 @@ pub struct PushSummary {
     /// realistic case). Silently retrying forever would be the wrong shape.
     pub rejected: usize,
     pub errors: Vec<String>,
+    /// Files attached on this device that reached the server this cycle.
+    pub uploaded: usize,
+    /// Files that didn't. Their reasons are in `errors`; one the server refused for
+    /// good also carries its reason on the attachment and isn't tried again.
+    pub upload_failed: usize,
 }
 
 impl PushSummary {
@@ -47,6 +54,8 @@ impl PushSummary {
         self.noop += other.noop;
         self.rejected += other.rejected;
         self.errors.extend(other.errors);
+        self.uploaded += other.uploaded;
+        self.upload_failed += other.upload_failed;
     }
 }
 
@@ -136,9 +145,17 @@ pub struct PushResult {
 
 /// Everything waiting to go up, oldest edit first so a truncated batch still makes
 /// forward progress in a sensible order.
-pub fn collect(conn: &Connection, limit: usize) -> rusqlite::Result<Vec<Change>> {
+///
+/// `attachment_sync` is whether the server takes attachment and preview removals.
+/// Without it they stay queued here, untouched, until a server that does — an older
+/// one would answer "unknown entity" and the removal would read as a failure.
+pub fn collect(
+    conn: &Connection,
+    limit: usize,
+    attachment_sync: bool,
+) -> rusqlite::Result<Vec<Change>> {
     let mut out = Vec::new();
-    collect_deletes(conn, &mut out, limit)?;
+    collect_deletes(conn, &mut out, limit, attachment_sync)?;
     if out.len() < limit {
         collect_labels(conn, &mut out, limit)?;
     }
@@ -148,11 +165,21 @@ pub fn collect(conn: &Connection, limit: usize) -> rusqlite::Result<Vec<Change>>
     Ok(out)
 }
 
-fn collect_deletes(conn: &Connection, out: &mut Vec<Change>, limit: usize) -> rusqlite::Result<()> {
+fn collect_deletes(
+    conn: &Connection,
+    out: &mut Vec<Change>,
+    limit: usize,
+    attachment_sync: bool,
+) -> rusqlite::Result<()> {
+    // Filtered in SQL rather than skipped below, so held-back removals can't use up
+    // the LIMIT and starve the deletes that can go.
     let mut stmt = conn.prepare(
-        "SELECT entity, id, deleted_at FROM pending_deletes ORDER BY deleted_at LIMIT ?1",
+        "SELECT entity, id, deleted_at FROM pending_deletes
+          WHERE entity IN ('note', 'label')
+             OR (?2 AND entity IN ('attachment', 'preview'))
+          ORDER BY deleted_at LIMIT ?1",
     )?;
-    let rows = stmt.query_map(params![limit as i64], |r| {
+    let rows = stmt.query_map(params![limit as i64, attachment_sync], |r| {
         Ok((
             r.get::<_, String>(0)?,
             r.get::<_, String>(1)?,
@@ -161,11 +188,13 @@ fn collect_deletes(conn: &Connection, out: &mut Vec<Change>, limit: usize) -> ru
     })?;
     for row in rows {
         let (entity, id, deleted_at) = row?;
-        // Only 'note' and 'label' exist on the wire; anything else is a bug in a
-        // writer, and shipping it would earn a blanket rejection for the batch.
+        // Only these exist on the wire; anything else is a bug in a writer, and
+        // shipping it would earn a rejection that no retry could clear.
         let entity: &'static str = match entity.as_str() {
             "note" => "note",
             "label" => "label",
+            "attachment" => "attachment",
+            "preview" => "preview",
             _ => continue,
         };
         out.push(Change::delete(entity, id, deleted_at));
@@ -414,7 +443,9 @@ pub fn has_pending(conn: &Connection) -> rusqlite::Result<bool> {
         .query_row(
             "SELECT 1 FROM notes WHERE dirty = 1
               UNION ALL SELECT 1 FROM labels WHERE dirty = 1
-              UNION ALL SELECT 1 FROM pending_deletes LIMIT 1",
+              UNION ALL SELECT 1 FROM pending_deletes
+              UNION ALL SELECT 1 FROM attachments WHERE uploaded = 0 AND upload_error IS NULL
+              LIMIT 1",
             [],
             |r| r.get(0),
         )
@@ -434,22 +465,141 @@ pub fn pending_fingerprint(conn: &Connection) -> rusqlite::Result<Option<String>
     let fingerprint: String = conn.query_row(
         "SELECT (SELECT COUNT(*) || ':' || IFNULL(MAX(updated_at), '') FROM notes WHERE dirty = 1)
              || '|' || (SELECT COUNT(*) || ':' || IFNULL(MAX(updated_at), '') FROM labels WHERE dirty = 1)
-             || '|' || (SELECT COUNT(*) || ':' || IFNULL(MAX(deleted_at), '') FROM pending_deletes)",
+             || '|' || (SELECT COUNT(*) || ':' || IFNULL(MAX(deleted_at), '') FROM pending_deletes)
+             || '|' || (SELECT COUNT(*) FROM attachments WHERE uploaded = 0 AND upload_error IS NULL)",
         [],
         |r| r.get(0),
     )?;
-    Ok((fingerprint != "0:|0:|0:").then_some(fingerprint))
+    Ok((fingerprint != "0:|0:|0:|0").then_some(fingerprint))
+}
+
+/// A file attached on this device, ready to go up.
+#[derive(Debug, PartialEq, Eq)]
+pub struct PendingUpload {
+    pub note_id: String,
+    pub id: String,
+    pub filename: String,
+    pub mime: String,
+    pub sha256: String,
+}
+
+/// Files waiting to upload whose note the server already holds. A note still dirty
+/// after the push (its change was rejected) keeps its files back too: the server files
+/// an attachment under its note, and would answer 404 for one it doesn't have.
+pub fn pending_uploads(conn: &Connection) -> rusqlite::Result<Vec<PendingUpload>> {
+    let mut stmt = conn.prepare(
+        "SELECT a.note_id, a.id, IFNULL(a.filename, 'file'), a.mime, a.sha256
+           FROM attachments a JOIN notes n ON n.id = a.note_id
+          WHERE a.uploaded = 0 AND a.upload_error IS NULL AND a.sha256 IS NOT NULL
+            AND n.dirty = 0
+          ORDER BY a.note_id, a.position",
+    )?;
+    let rows = stmt.query_map([], |r| {
+        Ok(PendingUpload {
+            note_id: r.get(0)?,
+            id: r.get(1)?,
+            filename: r.get(2)?,
+            mime: r.get(3)?,
+            sha256: r.get(4)?,
+        })
+    })?;
+    rows.collect()
+}
+
+/// Record what became of one upload.
+fn settle_upload(
+    conn: &Connection,
+    id: &str,
+    result: &Result<(), UploadError>,
+) -> rusqlite::Result<()> {
+    match result {
+        Ok(()) => conn.execute(
+            "UPDATE attachments SET uploaded = 1, upload_error = NULL WHERE id = ?1",
+            params![id],
+        )?,
+        Err(UploadError::Refused(reason)) => conn.execute(
+            "UPDATE attachments SET upload_error = ?2 WHERE id = ?1",
+            params![id, reason],
+        )?,
+        Err(UploadError::Retry(_)) => 0,
+    };
+    Ok(())
+}
+
+/// Send the bytes of every file attached here whose note has landed.
+///
+/// One file failing never fails the cycle, the same as a download: the notes have
+/// already gone, and one unreachable or oversized file must not hold up every sync
+/// after it. A refusal is recorded on the attachment instead, and a passing failure
+/// is simply tried again next cycle.
+async fn upload_pending(
+    db: &Db,
+    blobs: &BlobStore,
+    base_url: &str,
+    token: &str,
+) -> Result<PushSummary, String> {
+    let wanted = {
+        let conn = db.0.lock().map_err(|e| e.to_string())?;
+        pending_uploads(&conn).map_err(|e| e.to_string())?
+    };
+    let mut summary = PushSummary::default();
+    for upload in wanted {
+        let result = match blobs.read(&upload.sha256) {
+            Some(bytes) => {
+                client::upload_attachment(
+                    base_url,
+                    token,
+                    &upload.note_id,
+                    &upload.id,
+                    &upload.filename,
+                    &upload.mime,
+                    &upload.sha256,
+                    bytes,
+                )
+                .await
+            }
+            // Nothing to send and nothing that could bring it back: the file was
+            // only ever on this device.
+            None => Err(UploadError::Refused(
+                "the file's bytes are missing from this device".to_string(),
+            )),
+        };
+        {
+            let conn = db.0.lock().map_err(|e| e.to_string())?;
+            settle_upload(&conn, &upload.id, &result).map_err(|e| e.to_string())?;
+        }
+        match result {
+            Ok(()) => summary.uploaded += 1,
+            Err(UploadError::Refused(reason)) | Err(UploadError::Retry(reason)) => {
+                log::warn!("attachment {}: {reason}", upload.id);
+                summary.upload_failed += 1;
+                summary
+                    .errors
+                    .push(format!("{} didn't upload: {reason}", upload.filename));
+            }
+        }
+    }
+    Ok(summary)
 }
 
 /// Send everything pending, in batches, applying each batch's results before the
-/// next is collected.
-pub async fn run(db: &Db, base_url: &str, token: &str) -> Result<PushSummary, String> {
+/// next is collected — then the files, once their notes are there to hold them.
+///
+/// `attachment_sync`: whether the server advertises it (see [`collect`]). Without
+/// it, uploads wait too.
+pub async fn run(
+    db: &Db,
+    blobs: &BlobStore,
+    base_url: &str,
+    token: &str,
+    attachment_sync: bool,
+) -> Result<PushSummary, String> {
     let mut total = PushSummary::default();
 
     loop {
         let batch = {
             let conn = db.0.lock().map_err(|e| e.to_string())?;
-            collect(&conn, BATCH).map_err(|e| e.to_string())?
+            collect(&conn, BATCH, attachment_sync).map_err(|e| e.to_string())?
         };
         if batch.is_empty() {
             break;
@@ -477,6 +627,10 @@ pub async fn run(db: &Db, base_url: &str, token: &str) -> Result<PushSummary, St
         }
     }
 
+    if attachment_sync {
+        total.absorb(upload_pending(db, blobs, base_url, token).await?);
+    }
+
     if total.rejected > 0 {
         log::warn!(
             "push: {} change(s) rejected by the server: {}",
@@ -485,13 +639,16 @@ pub async fn run(db: &Db, base_url: &str, token: &str) -> Result<PushSummary, St
         );
     }
     log::info!(
-        "push complete: {} sent ({} created, {} applied, {} kept, {} noop, {} rejected)",
+        "push complete: {} sent ({} created, {} applied, {} kept, {} noop, {} rejected), \
+         {} file(s) uploaded, {} not",
         total.sent,
         total.created,
         total.applied,
         total.kept,
         total.noop,
-        total.rejected
+        total.rejected,
+        total.uploaded,
+        total.upload_failed
     );
     Ok(total)
 }
@@ -548,7 +705,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "clean", 0);
         seed_note(&conn, "dirty", 1);
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         assert_eq!(batch.len(), 1);
         assert_eq!(batch[0].id, "dirty");
         assert_eq!(batch[0].op, "upsert");
@@ -573,7 +730,7 @@ mod tests {
             )
             .expect("seed membership");
         }
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         let note = batch.iter().find(|c| c.entity == "note").expect("note");
         assert_eq!(note.label_ids.as_deref(), Some(&["manual".to_string()][..]));
     }
@@ -583,7 +740,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "n1", 0);
         store::delete_forever(&conn, "n1").expect("delete");
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         assert_eq!(batch.len(), 1);
         assert_eq!(batch[0].op, "delete");
         assert_eq!(batch[0].entity, "note");
@@ -594,7 +751,7 @@ mod tests {
     fn applied_clears_dirty_and_records_the_revision() {
         let conn = db();
         seed_note(&conn, "n1", 1);
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         apply_results(&conn, &batch, &[ok("applied", Some(42))]).expect("apply");
         assert_eq!(dirty_count(&conn), 0);
         let rev: i64 = conn
@@ -611,7 +768,7 @@ mod tests {
         // every time; the following pull adopts the server's version instead.
         let conn = db();
         seed_note(&conn, "n1", 1);
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         let summary = apply_results(&conn, &batch, &[ok("kept", Some(99))]).expect("apply");
         assert_eq!(summary.kept, 1);
         assert_eq!(dirty_count(&conn), 0);
@@ -625,7 +782,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "n1", 1);
         state::set_cursor(&conn, 100).expect("cursor");
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         apply_results(&conn, &batch, &[ok("kept", Some(40))]).expect("apply");
         assert_eq!(state::read(&conn).expect("state").last_cursor, 39);
     }
@@ -635,7 +792,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "n1", 1);
         state::set_cursor(&conn, 10).expect("cursor");
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         apply_results(&conn, &batch, &[ok("kept", Some(40))]).expect("apply");
         assert_eq!(
             state::read(&conn).expect("state").last_cursor,
@@ -648,7 +805,7 @@ mod tests {
     fn rejected_stays_dirty_and_is_reported() {
         let conn = db();
         seed_note(&conn, "n1", 1);
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         let mut bad = ok("rejected", None);
         bad.error = Some("name in use".into());
         let summary = apply_results(&conn, &batch, &[bad]).expect("apply");
@@ -662,7 +819,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "n1", 0);
         store::delete_forever(&conn, "n1").expect("delete");
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         apply_results(&conn, &batch, &[ok("applied", Some(7))]).expect("apply");
         assert!(!has_pending(&conn).expect("pending"));
     }
@@ -673,7 +830,7 @@ mod tests {
         let conn = db();
         seed_note(&conn, "n1", 1);
         store::delete_forever(&conn, "n1").expect("delete");
-        let batch = collect(&conn, 100).expect("collect");
+        let batch = collect(&conn, 100, true).expect("collect");
         apply_results(&conn, &batch, &[ok("noop", None)]).expect("apply");
         assert!(!has_pending(&conn).expect("pending"));
     }
@@ -761,4 +918,162 @@ mod tests {
         conn.execute("UPDATE notes SET dirty = 0", []).unwrap();
         assert_eq!(pending_fingerprint(&conn).unwrap(), None);
     }
+
+    fn queued_upload(conn: &Connection, id: &str, note_id: &str, error: Option<&str>) {
+        conn.execute(
+            "INSERT INTO attachments (id, note_id, url, filename, mime, sha256, uploaded, upload_error)
+                  VALUES (?1, ?2, '/x', 'f.txt', 'text/plain', 'h', 0, ?3)",
+            params![id, note_id, error],
+        )
+        .expect("queued upload");
+    }
+
+    fn upload_ids(conn: &Connection) -> Vec<String> {
+        pending_uploads(conn)
+            .expect("uploads")
+            .into_iter()
+            .map(|u| u.id)
+            .collect()
+    }
+
+    #[test]
+    fn removals_of_files_and_previews_wait_for_a_server_that_takes_them() {
+        let conn = db();
+        store::record_pending_delete(&conn, "attachment", "a1").expect("tombstone");
+        store::record_pending_delete(&conn, "preview", "p1").expect("tombstone");
+        store::record_pending_delete(&conn, "note", "n9").expect("tombstone");
+
+        // An older server would answer "unknown entity" to each of them.
+        let older: Vec<_> = collect(&conn, 100, false)
+            .expect("collect")
+            .iter()
+            .map(|c| c.entity)
+            .collect();
+        assert_eq!(older, vec!["note"]);
+
+        let mut newer: Vec<_> = collect(&conn, 100, true)
+            .expect("collect")
+            .iter()
+            .map(|c| (c.entity, c.op))
+            .collect();
+        newer.sort();
+        assert_eq!(
+            newer,
+            vec![
+                ("attachment", "delete"),
+                ("note", "delete"),
+                ("preview", "delete")
+            ]
+        );
+    }
+
+    #[test]
+    fn a_removed_attachment_stays_removed_through_a_whole_cycle() {
+        use crate::sync::{pull, wire};
+        let conn = db();
+        let server_note = |revision: i64, attachments: Vec<wire::Attachment>| wire::Note {
+            id: "n1".into(),
+            body: "a note".into(),
+            position: 0,
+            pinned: false,
+            archived: false,
+            trashed: false,
+            deleted_at: None,
+            remind_at: None,
+            recurrence: None,
+            created_at: Some("2026-07-26T00:00:00.000Z".into()),
+            updated_at: Some("2026-07-26T00:00:00.000Z".into()),
+            sync_revision: revision,
+            purged_at: None,
+            labels: vec![],
+            attachments,
+            previews: vec![],
+        };
+        let page = |note: wire::Note, cursor: i64| wire::ChangesPage {
+            notes: vec![note],
+            labels: vec![],
+            cursor,
+            has_more: false,
+        };
+        let a1 = wire::Attachment {
+            id: "a1".into(),
+            url: "/x".into(),
+            filename: None,
+            mime: "image/png".into(),
+            size: None,
+            sha256: None,
+        };
+        pull::apply_page(&conn, &page(server_note(1, vec![a1]), 1)).expect("first pull");
+
+        store::delete_attachment(&conn, "n1", "a1").expect("remove");
+
+        // Push: the removal and the touched note go up, and the server applies both.
+        let batch = collect(&conn, 100, true).expect("collect");
+        let results: Vec<PushResult> = batch
+            .iter()
+            .map(|c| ok("applied", (c.entity == "note").then_some(5)))
+            .collect();
+        apply_results(&conn, &batch, &results).expect("results");
+        assert!(
+            !has_pending(&conn).expect("pending"),
+            "the tombstone is settled"
+        );
+
+        // Pull: the server's note now comes without it.
+        pull::apply_page(&conn, &page(server_note(6, vec![]), 6)).expect("second pull");
+        let left: i64 = conn
+            .query_row("SELECT COUNT(*) FROM attachments", [], |r| r.get(0))
+            .expect("count");
+        assert_eq!(left, 0);
+    }
+
+    #[test]
+    fn a_file_uploads_only_once_its_note_is_on_the_server() {
+        let conn = db();
+        seed_note(&conn, "landed", 0);
+        seed_note(&conn, "unsent", 1);
+        queued_upload(&conn, "a", "landed", None);
+        queued_upload(&conn, "b", "unsent", None);
+        queued_upload(&conn, "c", "landed", Some("file is too large (max 25 MB)"));
+        assert_eq!(upload_ids(&conn), vec!["a"]);
+    }
+
+    #[test]
+    fn a_refused_upload_leaves_the_queue_and_a_passing_failure_stays_in_it() {
+        let conn = db();
+        seed_note(&conn, "n", 0);
+        queued_upload(&conn, "big", "n", None);
+        queued_upload(&conn, "flaky", "n", None);
+        assert!(
+            pending_fingerprint(&conn).expect("fp").is_some(),
+            "uploads are pending work"
+        );
+
+        let refused = Err(UploadError::Refused("file is too large (max 25 MB)".into()));
+        settle_upload(&conn, "big", &refused).expect("settle");
+        settle_upload(&conn, "flaky", &Err(UploadError::Retry("offline".into()))).expect("settle");
+        assert_eq!(
+            upload_ids(&conn),
+            vec!["flaky"],
+            "the refusal is not resent every cycle"
+        );
+        assert!(has_pending(&conn).expect("pending"));
+
+        settle_upload(&conn, "flaky", &Ok(())).expect("settle");
+        assert!(upload_ids(&conn).is_empty());
+        assert!(!has_pending(&conn).expect("pending"));
+        assert_eq!(pending_fingerprint(&conn).expect("fp"), None);
+        let reason: Option<String> = conn
+            .query_row(
+                "SELECT upload_error FROM attachments WHERE id = 'big'",
+                [],
+                |r| r.get(0),
+            )
+            .expect("row");
+        assert_eq!(
+            reason.as_deref(),
+            Some("file is too large (max 25 MB)"),
+            "shown on the file"
+        );
+    }
 }