M12 — the Android client, end to end #2

Merged
bvandeusen merged 86 commits from dev into main 2026-08-21 08:53:58 -04:00
10 changed files with 297 additions and 9 deletions
Showing only changes of commit ed623a7bef - Show all commits
+4
View File
@@ -38,6 +38,10 @@ log = "0.4"
# native-tls uses OpenSSL, whose headers (libssl-dev) ci-tauri already ships.
# default-features off drops http2/charset we don't need for a JSON API.
reqwest = { version = "0.12", default-features = false, features = ["json", "native-tls"] }
# Verifying downloaded attachment bytes against the sha256 the server advertised.
# Pure Rust (no C/asm beyond optional cpufeatures), so it costs the Windows
# cross-compile lane nothing — see ci-requirements.md on why that matters here.
sha2 = "0.10"
# Tauri's default release profile: smaller, faster shipped binaries.
[profile.release]
+5
View File
@@ -46,6 +46,11 @@ pub fn run() {
let db = local::open(&db_path)?;
log::info!("local store ready — {}", local::summary(&db));
app.manage(db);
// Attachment bytes live beside the database, filed by content hash, so a
// synced image is readable with no network (M10.7d).
let blobs = sync::blobs::BlobStore::new(dir.join("blobs"))?;
log::info!("attachment store ready: {}", blobs.root().display());
app.manage(blobs);
Ok(())
})
.invoke_handler(tauri::generate_handler![
+148
View File
@@ -0,0 +1,148 @@
//! Local storage for attachment bytes (M10.7d).
//!
//! Content-addressed: a blob is filed under its own sha256, so the same image
//! attached to five notes is stored once and re-downloading it is free. The hash is
//! also the integrity check — bytes that don't hash to what the server advertised
//! are refused rather than filed under a name that lies about them.
//!
//! Attachment METADATA rides the delta feed; only the bytes come through here
//! (docs/sync.md).
use std::fs;
use std::path::{Path, PathBuf};
/// A sha256 in lowercase hex, and nothing else.
///
/// This is a **path-safety** check, not a formatting nicety: the hash is taken
/// straight from a server response and used as a filename. Without it, a hostile or
/// buggy server could send `../../…` and steer a write outside the blob directory.
fn is_hash(candidate: &str) -> bool {
candidate.len() == 64 && candidate.bytes().all(|b| b.is_ascii_hexdigit())
}
fn digest(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(bytes);
hasher
.finalize()
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
pub struct BlobStore {
root: PathBuf,
}
impl BlobStore {
/// Open (creating if needed) the blob directory.
pub fn new(root: PathBuf) -> std::io::Result<Self> {
fs::create_dir_all(&root)?;
Ok(Self { root })
}
pub fn root(&self) -> &Path {
&self.root
}
/// Where a blob lives, or `None` if the hash isn't one.
pub fn path(&self, sha256: &str) -> Option<PathBuf> {
let lower = sha256.to_ascii_lowercase();
is_hash(&lower).then(|| self.root.join(lower))
}
/// Whether we already hold these bytes. Drives the "don't download it twice"
/// skip, which is the entire point of keying by content.
pub fn has(&self, sha256: &str) -> bool {
self.path(sha256).is_some_and(|p| p.is_file())
}
/// File bytes under `expected`, refusing them if they don't hash to it.
///
/// Verifying on the way IN rather than on the way out means a corrupted transfer
/// can never be served later as if it were genuine — and the next sync simply
/// tries again, because the blob still counts as missing.
pub fn store(&self, expected: &str, bytes: &[u8]) -> Result<PathBuf, String> {
let path = self
.path(expected)
.ok_or_else(|| format!("refusing an attachment with a malformed hash: {expected}"))?;
let actual = digest(bytes);
if actual != expected.to_ascii_lowercase() {
return Err(format!(
"attachment failed its integrity check (expected {expected}, got {actual})"
));
}
fs::write(&path, bytes).map_err(|e| format!("couldn't save an attachment: {e}"))?;
Ok(path)
}
pub fn read(&self, sha256: &str) -> Option<Vec<u8>> {
fs::read(self.path(sha256)?).ok()
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A blob store in a throwaway directory. No tempfile dependency for one test
/// fixture — the process id keeps concurrent runs apart.
fn store(tag: &str) -> BlobStore {
let dir = std::env::temp_dir().join(format!("ts-blobs-{}-{tag}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
BlobStore::new(dir).expect("store")
}
/// sha256("hello") — a fixed vector, so a broken digest can't agree with itself.
const HELLO: &str = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
#[test]
fn digest_matches_a_known_vector() {
assert_eq!(digest(b"hello"), HELLO);
}
#[test]
fn stores_and_reads_back() {
let store = store("roundtrip");
assert!(!store.has(HELLO));
store.store(HELLO, b"hello").expect("store");
assert!(store.has(HELLO));
assert_eq!(store.read(HELLO).as_deref(), Some(&b"hello"[..]));
}
#[test]
fn refuses_bytes_that_dont_match_the_hash() {
// A corrupted or substituted transfer must never be filed under a name that
// claims it's genuine.
let store = store("mismatch");
let err = store.store(HELLO, b"goodbye").expect_err("must reject");
assert!(err.contains("integrity"), "got {err}");
assert!(!store.has(HELLO), "nothing should have been written");
}
#[test]
fn rejects_a_hash_that_could_escape_the_directory() {
// The hash arrives from a server response and becomes a filename.
let store = store("traversal");
assert!(store.path("../../etc/passwd").is_none());
assert!(store.store("../../etc/passwd", b"x").is_err());
assert!(store.path("").is_none());
assert!(store.path("nothex!!").is_none());
}
#[test]
fn accepts_an_uppercase_hash() {
// The wire format isn't guaranteed to be lowercase; the filename is.
let store = store("case");
store.store(&HELLO.to_ascii_uppercase(), b"hello").expect("store");
assert!(store.has(HELLO), "should be found under the lowercase name");
}
#[test]
fn missing_blob_reads_as_none() {
let store = store("missing");
assert!(store.read(HELLO).is_none());
assert!(!store.has(HELLO));
}
}
+32
View File
@@ -223,6 +223,38 @@ pub async fn fetch_changes(
.map_err(|e| format!("Couldn't read the change feed from {base_url}: {e}"))
}
/// Download one attachment's bytes.
///
/// Metadata already arrived on the delta feed; this is only the payload, fetched
/// over the same route the web app uses (owner/shared scoped server-side).
pub async fn fetch_attachment(
base_url: &str,
token: &str,
note_id: &str,
attachment_id: &str,
) -> Result<Vec<u8>, String> {
let url = format!("{base_url}/api/notes/{note_id}/attachments/{attachment_id}");
let request = prepare(http_with(SYNC_TIMEOUT)?.get(url), Some(token));
let response = request
.send()
.await
.map_err(|e| describe_transport_error(base_url, &e))?;
let status = response.status();
if status == StatusCode::UNAUTHORIZED {
return Err(TOKEN_REJECTED.to_string());
}
if !status.is_success() {
return Err(unexpected_status(base_url, status));
}
response
.bytes()
.await
.map(|b| b.to_vec())
.map_err(|e| format!("Couldn't download an attachment from {base_url}: {e}"))
}
/// Send a batch of changes and hand back the raw reply.
///
/// Returns text rather than parsed results so this module stays pure transport —
+6 -2
View File
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
use tauri::State;
use crate::local::Db;
use crate::sync::blobs::BlobStore;
use crate::sync::client::{self, Identity, ProbeResult};
use crate::sync::compat::Compatibility;
use crate::sync::engine;
@@ -142,9 +143,12 @@ fn credentials(db: &State<'_, Db>) -> Result<(String, String), String> {
/// separately inside the crate, but offering a bare "pull" would let the UI overwrite
/// unsent local edits — the ordering isn't a suggestion, it's what keeps them.
#[tauri::command]
pub async fn sync_now(db: State<'_, Db>) -> Result<engine::SyncOutcome, String> {
pub async fn sync_now(
db: State<'_, Db>,
blobs: State<'_, BlobStore>,
) -> Result<engine::SyncOutcome, String> {
let (base_url, token) = credentials(&db)?;
engine::run_cycle(db.inner(), &base_url, &token).await
engine::run_cycle(db.inner(), blobs.inner(), &base_url, &token).await
}
/// Whether anything is waiting to be sent. Lets the UI show an honest "unsynced
+8 -2
View File
@@ -7,6 +7,7 @@
use chrono::{SecondsFormat, Utc};
use serde::Serialize;
use super::blobs::BlobStore;
use super::pull;
use super::push;
use super::state;
@@ -31,9 +32,14 @@ pub struct SyncOutcome {
/// A failed push aborts before the pull. Pulling anyway would take the exact rows we
/// just failed to save and overwrite them — turning a recoverable network error into
/// lost work.
pub async fn run_cycle(db: &Db, base_url: &str, token: &str) -> Result<SyncOutcome, String> {
pub async fn run_cycle(
db: &Db,
blobs: &BlobStore,
base_url: &str,
token: &str,
) -> Result<SyncOutcome, String> {
let push = push::run(db, base_url, token).await?;
let pull = pull::run(db, base_url, token).await?;
let pull = pull::run(db, blobs, base_url, token).await?;
if pull.clobbered_dirty > 0 {
// Push ran first and reported success, so nothing should still have been
+1
View File
@@ -12,6 +12,7 @@
//! The engine that moves notes — push, pull, last-write-wins — lands in M10.7b/c and
//! consults `compat` before it does anything.
pub mod blobs;
pub mod client;
pub mod commands;
pub mod compat;
+79 -1
View File
@@ -10,6 +10,7 @@ use chrono::{SecondsFormat, Utc};
use rusqlite::{params, Connection, OptionalExtension};
use serde::Serialize;
use super::blobs::BlobStore;
use super::client;
use super::state;
use super::wire;
@@ -33,6 +34,10 @@ pub struct PullSummary {
/// top. Should be 0 in the normal cycle, because push runs first; anything higher
/// means local work was overwritten, which is worth saying out loud.
pub clobbered_dirty: usize,
pub blobs_downloaded: usize,
/// Attachments whose bytes couldn't be fetched or failed verification. Counted
/// rather than fatal — see `download_missing_blobs`.
pub blobs_failed: usize,
}
impl PullSummary {
@@ -43,10 +48,66 @@ impl PullSummary {
self.labels_applied += other.labels_applied;
self.labels_deleted += other.labels_deleted;
self.clobbered_dirty += other.clobbered_dirty;
self.blobs_downloaded += other.blobs_downloaded;
self.blobs_failed += other.blobs_failed;
self.cursor = other.cursor;
}
}
/// `(note_id, attachment_id, sha256)` for every attachment that advertises a hash.
/// The caller filters against the blob store — which blobs we hold isn't a SQL
/// question.
pub fn hashed_attachments(conn: &Connection) -> rusqlite::Result<Vec<(String, String, String)>> {
let mut stmt = conn.prepare(
"SELECT note_id, id, sha256 FROM attachments
WHERE sha256 IS NOT NULL AND sha256 <> ''",
)?;
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
rows.collect()
}
/// Fetch the bytes for any attachment we have metadata for but no blob.
///
/// A failed attachment NEVER fails the sync. Notes are the primary data and they've
/// already landed; an image that didn't arrive is retried on the next cycle simply
/// because its blob still counts as missing. Aborting here would mean one unreachable
/// file could block every future sync.
async fn download_missing_blobs(
db: &Db,
blobs: &BlobStore,
base_url: &str,
token: &str,
) -> Result<(usize, usize), String> {
let wanted = {
let conn = db.0.lock().map_err(|e| e.to_string())?;
hashed_attachments(&conn).map_err(|e| e.to_string())?
};
let mut downloaded = 0;
let mut failed = 0;
for (note_id, attachment_id, sha256) in wanted {
// Content-addressed, so this skips blobs we already hold — including the same
// image attached to a different note.
if blobs.has(&sha256) {
continue;
}
match client::fetch_attachment(base_url, token, &note_id, &attachment_id).await {
Ok(bytes) => match blobs.store(&sha256, &bytes) {
Ok(_) => downloaded += 1,
Err(e) => {
log::warn!("attachment {attachment_id}: {e}");
failed += 1;
}
},
Err(e) => {
log::warn!("attachment {attachment_id}: {e}");
failed += 1;
}
}
}
Ok((downloaded, failed))
}
fn now() -> String {
Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true)
}
@@ -339,7 +400,12 @@ fn position_of(explicit: i64, index: usize) -> i64 {
/// NOTE ON ORDERING: the full cycle is push-then-pull (docs/sync.md). Running this
/// against a store with unpushed edits lets the server's version land on top of them
/// — counted as `clobbered_dirty` and logged, rather than hidden.
pub async fn run(db: &Db, base_url: &str, token: &str) -> Result<PullSummary, String> {
pub async fn run(
db: &Db,
blobs: &BlobStore,
base_url: &str,
token: &str,
) -> Result<PullSummary, String> {
let mut total = PullSummary::default();
loop {
@@ -377,12 +443,24 @@ pub async fn run(db: &Db, base_url: &str, token: &str) -> Result<PullSummary, St
}
}
// Notes first, bytes after: the metadata is what makes the attachments knowable,
// and knowing one is missing is what lets the next cycle retry it.
let (downloaded, failed) = download_missing_blobs(db, blobs, base_url, token).await?;
total.blobs_downloaded = downloaded;
total.blobs_failed = failed;
if total.clobbered_dirty > 0 {
log::warn!(
"pull overwrote {} note(s) that still had unpushed local edits",
total.clobbered_dirty
);
}
if total.blobs_failed > 0 {
log::warn!(
"pull: {} attachment(s) couldn't be downloaded; will retry next sync",
total.blobs_failed
);
}
log::info!(
"pull complete: {} page(s), {} note(s) applied, {} deleted, {} label(s) applied, cursor {}",
total.pages,
+3
View File
@@ -114,6 +114,9 @@ export interface PullSummary {
labels_deleted: number;
cursor: number;
clobbered_dirty: number;
blobs_downloaded: number;
/** Attachments whose bytes didn't arrive. Retried next sync, never fatal. */
blobs_failed: number;
}
export interface SyncOutcome {
+11 -4
View File
@@ -119,10 +119,17 @@ async function syncNow() {
pending.value = await syncBridge.hasPending();
const received = outcome.pull.notes_applied + outcome.pull.notes_deleted;
const sent = outcome.push.created + outcome.push.applied;
lastResult.value =
received === 0 && sent === 0
? "Already up to date."
: `Sent ${sent}, received ${received}.`;
const blobs = outcome.pull.blobs_downloaded;
const parts: string[] = [];
if (sent > 0) parts.push(`sent ${sent}`);
if (received > 0) parts.push(`received ${received}`);
if (blobs > 0) parts.push(`${blobs} attachment${blobs === 1 ? "" : "s"}`);
lastResult.value = parts.length ? `Synced — ${parts.join(", ")}.` : "Already up to date.";
// Attachments that didn't arrive are retried next sync, so this is a note, not
// an error — but saying nothing would leave a missing image unexplained.
if (outcome.pull.blobs_failed > 0) {
lastResult.value += ` ${outcome.pull.blobs_failed} attachment(s) didn't download — they'll retry on the next sync.`;
}
// Rejections are the server refusing a specific change — surfaced, never
// swallowed, because only the person can resolve them.
if (outcome.push.rejected > 0) {