feat(search): unify session transcript search (#2877)

src/agent_loop.py

@@ -332,7 +332,7 @@ If the user asks for a reminder/alarm before the event, pass `reminder_minutes`
     "create_session": "- ```create_session``` — Create a new chat. Line 1 = chat name, line 2 = model name. Use for background/parallel work.",
     "list_sessions": "- ```list_sessions``` — List chats sorted MOST-RECENT FIRST (the UI calls them 'chats') with clickable chat-title links. Output includes a relative \"last active\" timestamp per row, so the first row is the user's most recent chat. Content = optional filter keyword (matches chat name). When answering, preserve the `[title](#session-id)` links exactly; do not convert them into plain text.",
     "send_to_session": "- ```send_to_session``` — Send a message to another session. Line 1 = session_id, rest = message. Use for orchestrating work across sessions.",
-    "search_chats": "- ```search_chats``` — Search across all chat history. Use when user asks 'did we discuss X?' or 'find the conversation about Y'.",
+    "search_chats": "- ```search_chats``` — Search past session transcripts for direct conversation evidence. Use when user asks 'did we discuss X?', 'find the conversation about Y', or when prior chat context is more appropriate than persistent memory.",
     "pipeline": "- ```pipeline``` — Run a multi-step AI pipeline. Args (JSON) with ordered steps, each specifying a model and prompt. Use for complex workflows.",
     "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle <name> on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel <name>` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply <uid> <folder> <reply|reply-all|ai-reply>` (opens an email compose document, does NOT send), `set_mode agent/chat`, `switch_model <name>`, `set_theme <preset>`, `create_theme <name> <bg> <fg> <panel> <border> <accent>` (optional key=val for advanced colors AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel <name>`. Theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute.",
     "ask_user": "- ```ask_user``` — Ask the user a multiple-choice question when the task is genuinely ambiguous and the answer changes what you do next (pick an approach, confirm an assumption, choose a target). Args (JSON): {\"question\": \"...\", \"options\": [{\"label\": \"...\", \"description\": \"...\"?}, ...], \"multi\": false?}. 2-6 options. The user gets clickable buttons; calling this ENDS your turn and their choice comes back as your next message. Prefer sensible defaults — only ask when you truly can't proceed well without their input.",

src/session_search.py

@@ -0,0 +1,355 @@
+"""Shared session transcript search for UI and agent tools."""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Iterable
+
+from sqlalchemy import text
+
+from core.database import ChatMessage as DBChatMessage
+from core.database import Session as DBSession
+from core.database import SessionLocal
+
+logger = logging.getLogger(__name__)
+
+SEARCH_ROLES = ("user", "assistant")
+
+
+@dataclass(frozen=True)
+class SessionSearchResult:
+    message_id: str
+    session_id: str
+    session_name: str
+    role: str
+    content: str
+    content_snippet: str
+    timestamp: str | None
+    context_before: list[dict[str, Any]]
+    context_after: list[dict[str, Any]]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "message_id": self.message_id,
+            "session_id": self.session_id,
+            "session_name": self.session_name,
+            "role": self.role,
+            "content_snippet": self.content_snippet,
+            "timestamp": self.timestamp,
+            "context_before": self.context_before,
+            "context_after": self.context_after,
+        }
+
+
+def _iso(value: datetime | None) -> str | None:
+    return value.isoformat() if value else None
+
+
+def _message_to_context(msg: DBChatMessage) -> dict[str, Any]:
+    return {
+        "message_id": msg.id,
+        "role": msg.role,
+        "content": msg.content or "",
+        "timestamp": _iso(msg.timestamp),
+    }
+
+
+def _escape_like(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+def _snippet(content: str, query: str, radius: int = 60) -> str:
+    content = content or ""
+    query = query or ""
+    if not query:
+        return content[: radius * 2]
+
+    idx = content.lower().find(query.lower())
+    if idx == -1:
+        return content[: radius * 2]
+
+    start = max(0, idx - radius)
+    end = min(len(content), idx + len(query) + radius)
+    return ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
+
+
+def _sanitize_fts_query(query: str) -> str | None:
+    """Convert free text into a conservative FTS5 MATCH query.
+
+    User input can contain FTS5 operators or punctuation that raises
+    sqlite3.OperationalError. For transcript search we do not need advanced
+    syntax in v1, so keep only words and balanced quoted phrases.
+    """
+    parts: list[str] = []
+    for match in re.finditer(r'"([^"]+)"|[\w][\w._-]*', query, flags=re.UNICODE):
+        phrase = match.group(1)
+        if phrase is not None:
+            phrase = phrase.strip()
+            if phrase:
+                parts.append('"' + phrase.replace('"', '""') + '"')
+            continue
+
+        token = match.group(0).strip("._-")
+        if not token:
+            continue
+        if any(ch in token for ch in "._-"):
+            parts.append('"' + token.replace('"', '""') + '"')
+        else:
+            parts.append(token)
+
+    if not parts:
+        return None
+    return " ".join(parts)
+
+
+def _is_sqlite_session(db) -> bool:
+    try:
+        bind = db.get_bind()
+        return getattr(getattr(bind, "dialect", None), "name", None) == "sqlite"
+    except Exception:
+        return False
+
+
+def _has_fts_table(db) -> bool:
+    if not _is_sqlite_session(db):
+        return False
+    try:
+        row = db.execute(
+            text("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chat_messages_fts' LIMIT 1")
+        ).first()
+        return row is not None
+    except Exception as e:
+        logger.debug("chat_messages_fts availability check failed: %s", e)
+        return False
+
+
+def _owner_filter(query, owner: str | None, include_legacy_owner: bool):
+    if owner is None:
+        return query.filter(DBSession.owner.is_(None))
+    if not include_legacy_owner:
+        return query.filter(DBSession.owner == owner)
+    return query.filter((DBSession.owner == owner) | (DBSession.owner.is_(None)))
+
+
+def _context_for_message(db, msg: DBChatMessage, count: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    if count <= 0 or not msg.timestamp:
+        return [], []
+
+    before_rows = (
+        db.query(DBChatMessage)
+        .filter(
+            DBChatMessage.session_id == msg.session_id,
+            DBChatMessage.role.in_(SEARCH_ROLES),
+            DBChatMessage.timestamp < msg.timestamp,
+        )
+        .order_by(DBChatMessage.timestamp.desc())
+        .limit(count)
+        .all()
+    )
+    after_rows = (
+        db.query(DBChatMessage)
+        .filter(
+            DBChatMessage.session_id == msg.session_id,
+            DBChatMessage.role.in_(SEARCH_ROLES),
+            DBChatMessage.timestamp > msg.timestamp,
+        )
+        .order_by(DBChatMessage.timestamp.asc())
+        .limit(count)
+        .all()
+    )
+    before = [_message_to_context(row) for row in reversed(before_rows)]
+    after = [_message_to_context(row) for row in after_rows]
+    return before, after
+
+
+def _rows_to_results(db, rows: Iterable[tuple[DBChatMessage, str, str]], query: str, context_messages: int) -> list[SessionSearchResult]:
+    results: list[SessionSearchResult] = []
+    for msg, session_name, snippet in rows:
+        before, after = _context_for_message(db, msg, context_messages)
+        content = msg.content or ""
+        results.append(
+            SessionSearchResult(
+                message_id=msg.id,
+                session_id=msg.session_id,
+                session_name=session_name or "Untitled",
+                role=msg.role,
+                content=content,
+                content_snippet=snippet or _snippet(content, query),
+                timestamp=_iso(msg.timestamp),
+                context_before=before,
+                context_after=after,
+            )
+        )
+    return results
+
+
+def _search_like(
+    db,
+    query: str,
+    limit: int,
+    owner: str | None,
+    include_archived: bool,
+    context_messages: int,
+    restrict_owner: bool,
+    include_legacy_owner: bool,
+) -> list[SessionSearchResult]:
+    safe_q = _escape_like(query)
+    q = (
+        db.query(DBChatMessage, DBSession.name)
+        .join(DBSession, DBChatMessage.session_id == DBSession.id)
+        .filter(
+            DBChatMessage.content.ilike(f"%{safe_q}%", escape="\\"),
+            DBChatMessage.role.in_(SEARCH_ROLES),
+        )
+    )
+    if not include_archived:
+        q = q.filter(DBSession.archived == False)
+    if restrict_owner:
+        q = _owner_filter(q, owner, include_legacy_owner)
+    rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
+    shaped = ((msg, session_name, _snippet(msg.content or "", query)) for msg, session_name in rows)
+    return _rows_to_results(db, shaped, query, context_messages)
+
+
+def _search_fts(
+    db,
+    query: str,
+    limit: int,
+    owner: str | None,
+    include_archived: bool,
+    context_messages: int,
+    restrict_owner: bool,
+    include_legacy_owner: bool,
+) -> list[SessionSearchResult] | None:
+    fts_query = _sanitize_fts_query(query)
+    if not fts_query or not _has_fts_table(db):
+        return None
+
+    archived_clause = "" if include_archived else "AND s.archived = 0"
+    if not restrict_owner:
+        owner_clause = ""
+    elif owner is None:
+        owner_clause = "AND s.owner IS NULL"
+    elif not include_legacy_owner:
+        owner_clause = "AND s.owner = :owner"
+    else:
+        owner_clause = "AND (s.owner = :owner OR s.owner IS NULL)"
+    params: dict[str, Any] = {"fts_query": fts_query, "limit": limit}
+    if restrict_owner and owner is not None:
+        params["owner"] = owner
+
+    sql = text(
+        f"""
+        SELECT
+            m.id AS message_id,
+            snippet(chat_messages_fts, 0, '', '', '...', 24) AS content_snippet
+        FROM chat_messages_fts
+        JOIN chat_messages m ON m.id = chat_messages_fts.message_id
+        JOIN sessions s ON s.id = m.session_id
+        WHERE chat_messages_fts MATCH :fts_query
+          {archived_clause}
+          {owner_clause}
+          AND m.role IN ('user', 'assistant')
+        ORDER BY bm25(chat_messages_fts), m.timestamp DESC
+        LIMIT :limit
+        """
+    )
+
+    try:
+        hits = db.execute(sql, params).fetchall()
+    except Exception as e:
+        logger.debug("FTS session search failed; falling back to LIKE: %s", e)
+        return None
+
+    if not hits:
+        return None
+
+    rows = []
+    for hit in hits:
+        message_id = hit[0]
+        snippet = hit[1] or ""
+        row = (
+            db.query(DBChatMessage, DBSession.name)
+            .join(DBSession, DBChatMessage.session_id == DBSession.id)
+            .filter(DBChatMessage.id == message_id)
+            .first()
+        )
+        if row:
+            msg, session_name = row
+            rows.append((msg, session_name, snippet))
+    return _rows_to_results(db, rows, query, context_messages)
+
+
+def search_session_messages(
+    query: str,
+    limit: int = 20,
+    owner: str | None = None,
+    include_archived: bool = False,
+    context_messages: int = 1,
+    restrict_owner: bool = True,
+    include_legacy_owner: bool = True,
+    db=None,
+) -> list[SessionSearchResult]:
+    """Search session transcripts using FTS5 when available.
+
+    `owner=None` is deliberately treated as legacy/null-owner scope rather
+    than global access.
+    """
+    query = (query or "").strip()
+    if not query:
+        return []
+
+    limit = max(1, min(int(limit or 20), 100))
+    context_messages = max(0, min(int(context_messages or 0), 3))
+
+    owns_db = db is None
+    if owns_db:
+        db = SessionLocal()
+    try:
+        fts_results = _search_fts(
+            db,
+            query,
+            limit,
+            owner,
+            include_archived,
+            context_messages,
+            restrict_owner,
+            include_legacy_owner,
+        )
+        if fts_results is not None:
+            like_results = _search_like(
+                db,
+                query,
+                limit,
+                owner,
+                include_archived,
+                context_messages,
+                restrict_owner,
+                include_legacy_owner,
+            )
+            merged: list[SessionSearchResult] = []
+            seen: set[str] = set()
+            for result in [*fts_results, *like_results]:
+                if result.message_id in seen:
+                    continue
+                seen.add(result.message_id)
+                merged.append(result)
+                if len(merged) >= limit:
+                    break
+            return merged
+        return _search_like(
+            db,
+            query,
+            limit,
+            owner,
+            include_archived,
+            context_messages,
+            restrict_owner,
+            include_legacy_owner,
+        )
+    finally:
+        if owns_db:
+            db.close()

src/tool_implementations.py

@@ -548,7 +548,7 @@ async def do_suggest_document(content: str, doc_id: str = None, owner: Optional[
 # ---------------------------------------------------------------------------
 
 async def do_search_chats(query: str, limit: int = 20, owner: str | None = None) -> Dict:
-    """Search past chat messages for the calling user's sessions only.
+    """Search past session transcripts for the calling user's sessions only.
 
     Without an owner filter this used to leak EVERY user's chat history
     into the agent's `search_chats` results (v2 review HIGH-11). The
@@ -556,63 +556,36 @@ async def do_search_chats(query: str, limit: int = 20, owner: str | None = None)
     through; legacy callers without owner pass through as before but
     will only see legacy/null-owner rows.
     """
-    from src.database import SessionLocal, ChatMessage as DBChatMessage, Session as DBSession
-    # Escape LIKE wildcards in the user-supplied query so a stray % or _
-    # doesn't widen the match (and to keep the response deterministic).
-    safe_q = query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
-    db = SessionLocal()
     try:
-        q = (
-            db.query(DBChatMessage, DBSession.id, DBSession.name)
-            .join(DBSession, DBChatMessage.session_id == DBSession.id)
-            .filter(
-                DBSession.archived == False,
-                DBChatMessage.content.ilike(f"%{safe_q}%", escape="\\"),
-                DBChatMessage.role.in_(["user", "assistant"]),
-            )
-        )
-        if owner is not None:
-            # Restrict to this user's sessions plus legacy null-owner
-            # rows (so single-user upgrades keep seeing their own data).
-            q = q.filter((DBSession.owner == owner) | (DBSession.owner.is_(None)))
-        rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
+        from src.session_search import search_session_messages
 
-        if not rows:
+        results = search_session_messages(query, limit=limit, owner=owner)
+        if not results:
             return {"results": f"No chats found matching \"{query}\"."}
 
         # Group by session to avoid duplicate links
         seen_sessions = {}
-        for msg, session_id, session_name in rows:
-            if session_id not in seen_sessions:
-                content = msg.content or ""
-                lower_content = content.lower()
-                idx = lower_content.find(query.lower())
-                if idx == -1:
-                    snippet = content[:150]
-                else:
-                    start = max(0, idx - 60)
-                    end = min(len(content), idx + len(query) + 60)
-                    snippet = ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
-                seen_sessions[session_id] = {
-                    "name": session_name or "Untitled",
-                    "snippet": snippet,
-                    "role": msg.role,
-                    "timestamp": msg.timestamp.isoformat() if msg.timestamp else None,
-                }
+        for result in results:
+            if result.session_id not in seen_sessions:
+                seen_sessions[result.session_id] = result
 
         lines = [f"Found {len(seen_sessions)} session(s) matching \"{query}\":\n"]
-        for sid, info in seen_sessions.items():
-            lines.append(f"- **{info['name']}** (#{sid})")
+        for sid, result in seen_sessions.items():
+            lines.append(f"- **{result.session_name}** (#{sid})")
             lines.append(f"  Link: [Open chat](#{sid})")
-            lines.append(f"  > {info['snippet']}")
+            lines.append(f"  Match ({result.role}): {result.content_snippet}")
+            if result.context_before:
+                before = result.context_before[-1]
+                lines.append(f"  Before ({before['role']}): {before['content'][:180]}")
+            if result.context_after:
+                after = result.context_after[0]
+                lines.append(f"  After ({after['role']}): {after['content'][:180]}")
             lines.append("")
 
         return {"results": "\n".join(lines)}
     except Exception as e:
         logger.error(f"search_chats failed: {e}")
         return {"error": str(e), "exit_code": 1}
-    finally:
-        db.close()
 
 
 # ---------------------------------------------------------------------------

src/tool_index.py

@@ -115,7 +115,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
     "create_session": "Create a new chat with a name and model.",
     "list_sessions": "List all chats with their metadata (the UI calls these 'chats'). Use for 'list my chats', 'rename all my chats' (list first, then manage_session to rename each).",
     "send_to_session": "Send a message to another chat. Cross-chat communication.",
-    "search_chats": "Search through chat history across all sessions.",
+    "search_chats": "Search past session transcripts across chats.",
     "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.",
     "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.",
     "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",

src/tool_schemas.py

@@ -258,7 +258,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "search_chats",
-            "description": "Search the user's past chat conversations by keyword. Use when the user asks about previous chats, past conversations, or wants to find a discussion they had before. Returns matching sessions with clickable links.",
+            "description": "Search the user's past session transcripts by keyword. Use when the user asks about previous chats, past conversations, or when direct transcript evidence is better than persistent memory. Returns matching sessions with clickable links and nearby context.",
             "parameters": {
                 "type": "object",
                 "properties": {