Reading a range, for an export to walk

Adds the two queries an export needs and no more: `messages_between` and a
count beside it.

Paged by time rather than by offset, which is the part that matters. An
export reads a range while the paging beside it is still reaching further
back into the same table, and an OFFSET into a table growing underneath the
reader skips rows. "After the last one I saw" cannot, so the previous
page's last timestamp is what the next page is asked for.

No second backfill path: `getBacklog` with a `before` already pages older
history per service - Discord's snowflakes, Kick's cursor, IRC's
CHATHISTORY, Matrix's /messages token - each with its own pacing. The
export loop drives that, rather than growing a parallel implementation that
would have to learn the same four things again.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
2026-09-11 23:37:12 -04:00
co-authored by Claude Opus 5
parent 2e9bd0d7d3
commit f3a92e64f1
2 changed files with 79 additions and 0 deletions
+34
View File
@@ -2018,6 +2018,40 @@ pub async fn dispatch(
}
}
// A window of a conversation between two moments, oldest first.
//
// Paged by time rather than by offset, because an export reads a range
// that the paging beside it is still reaching back into - an OFFSET
// into a table growing underneath the reader skips rows, and "after
// the last one I saw" cannot. Pass the previous page's last timestamp
// back as `after`.
"getMessageRange" => {
let Some(buffer_id) = p_str_opt(params, "bufferId") else {
return (None, Some("getMessageRange requires \"bufferId\"".to_string()));
};
let after = p_i64(params, "after", 0);
let until = p_i64(params, "until", i64::MAX);
let limit = p_i64(params, "limit", 500);
match state.store.messages_between(buffer_id, after, until, limit) {
Ok(rows) => (Some(serde_json::to_value(rows).unwrap_or_default()), None),
Err(e) => (None, Some(e.to_string())),
}
}
// How much is held for a range, so an export can show a proportion
// rather than a count climbing towards nothing.
"countMessageRange" => {
let Some(buffer_id) = p_str_opt(params, "bufferId") else {
return (None, Some("countMessageRange requires \"bufferId\"".to_string()));
};
let after = p_i64(params, "after", 0);
let until = p_i64(params, "until", i64::MAX);
match state.store.count_between(buffer_id, after, until) {
Ok(n) => (Some(serde_json::json!({ "count": n, "oldestHeld": state.store.oldest_message_ts(buffer_id).ok().flatten() })), None),
Err(e) => (None, Some(e.to_string())),
}
}
// Writing a conversation to disk.
//
// Six calls rather than one, because the window drives the loop: the
+45
View File
@@ -614,6 +614,51 @@ impl Store {
Ok(rows.collect::<rusqlite::Result<_>>()?)
}
/// A window of conversation between two moments, oldest first.
///
/// Paged by time rather than by offset: an export walks forward through a
/// range that the backfill beside it may still be adding to, and an OFFSET
/// into a table that is growing underneath the reader skips rows. Asking
/// for "after the last one I saw" cannot.
///
/// `after` is exclusive and `until` inclusive, so passing the last
/// timestamp of one page as the `after` of the next neither repeats a
/// message nor drops one.
pub fn messages_between(&self, buffer_id: &str, after: i64, until: i64, limit: i64) -> Result<Vec<Message>> {
let conn = self.conn.lock().unwrap();
let limit = if limit > 0 { limit } else { 500 };
let mut stmt = conn.prepare(
"SELECT msg_id, from_nick, body, ts, is_action, is_highlight, kind, reply_to_id, reply_to_from, reply_to_body, edited, reactions, is_own, avatar_url, embeds, sender_id, attachments, html, sender_color, badges, reply_is_thread, reply_forwarded, components
FROM messages
WHERE buffer_id = ?1 AND ts > ?2 AND ts <= ?3
ORDER BY ts ASC LIMIT ?4",
)?;
let rows = stmt.query_map(params![buffer_id, after, until, limit], |row| Self::row_to_message(buffer_id, row))?;
Ok(rows.collect::<rusqlite::Result<_>>()?)
}
/// How many messages are held for a range, so an export can say how far
/// along it is before it starts rather than counting as it goes.
pub fn count_between(&self, buffer_id: &str, after: i64, until: i64) -> Result<i64> {
let conn = self.conn.lock().unwrap();
Ok(conn.query_row(
"SELECT COUNT(*) FROM messages WHERE buffer_id = ?1 AND ts > ?2 AND ts <= ?3",
params![buffer_id, after, until],
|r| r.get(0),
)?)
}
/// The oldest message held for a buffer, which is where a backfill has to
/// start reaching back from.
pub fn oldest_message_ts(&self, buffer_id: &str) -> Result<Option<i64>> {
let conn = self.conn.lock().unwrap();
Ok(conn
.query_row("SELECT MIN(ts) FROM messages WHERE buffer_id = ?1", params![buffer_id], |r| {
r.get::<_, Option<i64>>(0)
})
.unwrap_or(None))
}
/// The conversation either side of one moment, oldest first.
///
/// `get_backlog` only ever reads backwards, which is right for scrolling