Reading a range, for an export to walk
Adds the two queries an export needs and no more: `messages_between` and a count beside it. Paged by time rather than by offset, which is the part that matters. An export reads a range while the paging beside it is still reaching further back into the same table, and an OFFSET into a table growing underneath the reader skips rows. "After the last one I saw" cannot, so the previous page's last timestamp is what the next page is asked for. No second backfill path: `getBacklog` with a `before` already pages older history per service - Discord's snowflakes, Kick's cursor, IRC's CHATHISTORY, Matrix's /messages token - each with its own pacing. The export loop drives that, rather than growing a parallel implementation that would have to learn the same four things again. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
@@ -2018,6 +2018,40 @@ pub async fn dispatch(
|
||||
}
|
||||
}
|
||||
|
||||
// A window of a conversation between two moments, oldest first.
|
||||
//
|
||||
// Paged by time rather than by offset, because an export reads a range
|
||||
// that the paging beside it is still reaching back into - an OFFSET
|
||||
// into a table growing underneath the reader skips rows, and "after
|
||||
// the last one I saw" cannot. Pass the previous page's last timestamp
|
||||
// back as `after`.
|
||||
"getMessageRange" => {
|
||||
let Some(buffer_id) = p_str_opt(params, "bufferId") else {
|
||||
return (None, Some("getMessageRange requires \"bufferId\"".to_string()));
|
||||
};
|
||||
let after = p_i64(params, "after", 0);
|
||||
let until = p_i64(params, "until", i64::MAX);
|
||||
let limit = p_i64(params, "limit", 500);
|
||||
match state.store.messages_between(buffer_id, after, until, limit) {
|
||||
Ok(rows) => (Some(serde_json::to_value(rows).unwrap_or_default()), None),
|
||||
Err(e) => (None, Some(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
// How much is held for a range, so an export can show a proportion
|
||||
// rather than a count climbing towards nothing.
|
||||
"countMessageRange" => {
|
||||
let Some(buffer_id) = p_str_opt(params, "bufferId") else {
|
||||
return (None, Some("countMessageRange requires \"bufferId\"".to_string()));
|
||||
};
|
||||
let after = p_i64(params, "after", 0);
|
||||
let until = p_i64(params, "until", i64::MAX);
|
||||
match state.store.count_between(buffer_id, after, until) {
|
||||
Ok(n) => (Some(serde_json::json!({ "count": n, "oldestHeld": state.store.oldest_message_ts(buffer_id).ok().flatten() })), None),
|
||||
Err(e) => (None, Some(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
// Writing a conversation to disk.
|
||||
//
|
||||
// Six calls rather than one, because the window drives the loop: the
|
||||
|
||||
@@ -614,6 +614,51 @@ impl Store {
|
||||
Ok(rows.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
/// A window of conversation between two moments, oldest first.
|
||||
///
|
||||
/// Paged by time rather than by offset: an export walks forward through a
|
||||
/// range that the backfill beside it may still be adding to, and an OFFSET
|
||||
/// into a table that is growing underneath the reader skips rows. Asking
|
||||
/// for "after the last one I saw" cannot.
|
||||
///
|
||||
/// `after` is exclusive and `until` inclusive, so passing the last
|
||||
/// timestamp of one page as the `after` of the next neither repeats a
|
||||
/// message nor drops one.
|
||||
pub fn messages_between(&self, buffer_id: &str, after: i64, until: i64, limit: i64) -> Result<Vec<Message>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let limit = if limit > 0 { limit } else { 500 };
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT msg_id, from_nick, body, ts, is_action, is_highlight, kind, reply_to_id, reply_to_from, reply_to_body, edited, reactions, is_own, avatar_url, embeds, sender_id, attachments, html, sender_color, badges, reply_is_thread, reply_forwarded, components
|
||||
FROM messages
|
||||
WHERE buffer_id = ?1 AND ts > ?2 AND ts <= ?3
|
||||
ORDER BY ts ASC LIMIT ?4",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![buffer_id, after, until, limit], |row| Self::row_to_message(buffer_id, row))?;
|
||||
Ok(rows.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
/// How many messages are held for a range, so an export can say how far
|
||||
/// along it is before it starts rather than counting as it goes.
|
||||
pub fn count_between(&self, buffer_id: &str, after: i64, until: i64) -> Result<i64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
Ok(conn.query_row(
|
||||
"SELECT COUNT(*) FROM messages WHERE buffer_id = ?1 AND ts > ?2 AND ts <= ?3",
|
||||
params![buffer_id, after, until],
|
||||
|r| r.get(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// The oldest message held for a buffer, which is where a backfill has to
|
||||
/// start reaching back from.
|
||||
pub fn oldest_message_ts(&self, buffer_id: &str) -> Result<Option<i64>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
Ok(conn
|
||||
.query_row("SELECT MIN(ts) FROM messages WHERE buffer_id = ?1", params![buffer_id], |r| {
|
||||
r.get::<_, Option<i64>>(0)
|
||||
})
|
||||
.unwrap_or(None))
|
||||
}
|
||||
|
||||
/// The conversation either side of one moment, oldest first.
|
||||
///
|
||||
/// `get_backlog` only ever reads backwards, which is right for scrolling
|
||||
|
||||
Reference in New Issue
Block a user