mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-06 13:38:40 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6b8c3ae6b0 |
@@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service))
|
||||
logger.info("STT service initialized (provider managed via settings)")
|
||||
|
||||
# Documents (artifacts/canvas)
|
||||
from routes.document.document_routes import setup_document_routes
|
||||
from routes.document_routes import setup_document_routes
|
||||
document_router = setup_document_routes(session_manager, upload_handler)
|
||||
app.include_router(document_router)
|
||||
|
||||
@@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
|
||||
logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
|
||||
|
||||
# Webhooks
|
||||
from routes.webhook.webhook_routes import setup_webhook_routes
|
||||
from routes.webhook_routes import setup_webhook_routes
|
||||
app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
|
||||
|
||||
# API Tokens
|
||||
@@ -852,7 +852,7 @@ app.include_router(setup_codex_routes(
|
||||
))
|
||||
app.include_router(setup_claude_routes())
|
||||
|
||||
from routes.vault.vault_routes import setup_vault_routes
|
||||
from routes.vault_routes import setup_vault_routes
|
||||
app.include_router(setup_vault_routes())
|
||||
|
||||
# Contacts (CardDAV)
|
||||
|
||||
@@ -17,8 +17,6 @@ from mcp.types import Tool, TextContent
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
server = Server("memory")
|
||||
|
||||
# Late-initialized managers (set during first tool call)
|
||||
@@ -31,10 +29,6 @@ _OWNER_SCOPE_ERROR = (
|
||||
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
|
||||
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
|
||||
)
|
||||
_UNREADABLE_STORE_ERROR = (
|
||||
"Error: Memory store is temporarily unreadable — nothing was saved. "
|
||||
"Repair or restore memory.json, then retry."
|
||||
)
|
||||
|
||||
|
||||
def _configured_owner() -> str | None:
|
||||
@@ -57,20 +51,8 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
|
||||
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
|
||||
|
||||
|
||||
def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||
"""Return configured owner, all entries, visible entries, and optional error.
|
||||
|
||||
``for_update=True`` is for read-modify-write callers. They save the ``all
|
||||
entries`` list back, so an unreadable store must be reported as an error
|
||||
instead of degrading to ``[]`` — otherwise the save writes their one new
|
||||
entry over the whole store (issue #5673).
|
||||
"""
|
||||
if for_update:
|
||||
try:
|
||||
entries = _memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
|
||||
else:
|
||||
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||
"""Return configured owner, all entries, visible entries, and optional error."""
|
||||
entries = _memory_manager.load_all()
|
||||
owner = _configured_owner()
|
||||
if owner is None and _owner_scoped_store(entries):
|
||||
@@ -179,7 +161,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
category = arguments.get("category", "fact")
|
||||
if not text:
|
||||
return _text_result("Error: Memory text cannot be empty")
|
||||
owner, memories, _visible, scope_error = _scope_entries(for_update=True)
|
||||
owner, memories, _visible, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||
|
||||
@@ -33,4 +33,4 @@ PyMuPDF
|
||||
# magika (onnxruntime), already a core dep via fastembed. We avoid the
|
||||
# [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
|
||||
# the dependency-age discussion in issue #485.
|
||||
markitdown[docx,pptx,xlsx,xls]==0.1.6
|
||||
markitdown[docx,pptx,xlsx,xls]==0.1.7
|
||||
|
||||
+3
-3
@@ -3,9 +3,9 @@ uvicorn
|
||||
python-multipart
|
||||
python-dotenv
|
||||
httpx
|
||||
httpcore>=1.0,<2.0
|
||||
httpcore>=1.0.9,<2.0
|
||||
pydantic>=2.13.4
|
||||
pydantic-settings>=2.14.1
|
||||
pydantic-settings>=2.14.2
|
||||
SQLAlchemy
|
||||
pypdf
|
||||
beautifulsoup4
|
||||
@@ -41,7 +41,7 @@ bcrypt
|
||||
# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
|
||||
# breaking rewrite, so keep fresh installs on the maintained v1 line until the
|
||||
# servers are migrated together.
|
||||
mcp<2
|
||||
mcp<3
|
||||
pyotp
|
||||
qrcode[pil]
|
||||
croniter
|
||||
|
||||
+1
-10
@@ -6,7 +6,6 @@ from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request, Response
|
||||
from core.middleware import require_admin
|
||||
from services.memory import MemoryStoreUnreadable
|
||||
from src.auth_helpers import get_current_user
|
||||
from src.settings import load_settings, save_settings, load_features, save_features
|
||||
|
||||
@@ -77,15 +76,7 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
|
||||
|
||||
# ── Memories ──
|
||||
if "memories" in body and isinstance(body["memories"], list):
|
||||
# Strict load: importing on top of an unreadable store would write
|
||||
# only the incoming rows back and drop everything already saved.
|
||||
try:
|
||||
existing = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to import memories: %s", e)
|
||||
raise HTTPException(
|
||||
503, "Memory store is temporarily unreadable — nothing was imported."
|
||||
)
|
||||
existing = memory_manager.load_all()
|
||||
# Dedup against THIS user's own memories only. Using every tenant's
|
||||
# rows (load_all) meant a memory whose text matched any other
|
||||
# user's was silently skipped, so the importing user lost their own
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
"""Document route domain package (slice 2m, #4082/#4071).
|
||||
|
||||
Contains document_routes.py and document_helpers.py, migrated from the flat
|
||||
routes/ directory. Backward-compat shims at routes/document_routes.py and
|
||||
routes/document_helpers.py re-export from here.
|
||||
"""
|
||||
@@ -1,243 +0,0 @@
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
File diff suppressed because it is too large
Load Diff
+239
-10
@@ -1,14 +1,243 @@
|
||||
"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.document_helpers``, ``from routes.document_helpers import
|
||||
X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
|
||||
pattern used by test_security_regressions.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2m (#4082/#4071).
|
||||
"""
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import sys as _sys
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from routes.document import document_helpers as _canonical # noqa: F401
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
|
||||
+1806
-13
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str:
|
||||
return text
|
||||
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
||||
|
||||
from services.memory import MemoryManager, MemoryStoreUnreadable
|
||||
from services.memory import MemoryManager
|
||||
from core.session_manager import SessionManager
|
||||
from src.request_models import MemoryAddRequest
|
||||
from core.database import SessionLocal
|
||||
@@ -35,22 +35,6 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
|
||||
"""Load the whole store for a read-modify-write cycle.
|
||||
|
||||
A transient read failure must not look like an empty store: the caller
|
||||
would append to ``[]`` and save that back, atomically destroying every
|
||||
existing memory (issue #5673). Surface it as a 503 and change nothing.
|
||||
"""
|
||||
try:
|
||||
return memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to rewrite the memory store: %s", e)
|
||||
raise HTTPException(
|
||||
503, "Memory store is temporarily unreadable — no changes were made."
|
||||
)
|
||||
|
||||
|
||||
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
||||
"""Set up memory-related routes."""
|
||||
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
||||
@@ -132,7 +116,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
||||
if memory_data.session_id:
|
||||
new_entry["session_id"] = memory_data.session_id
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
all_mem.append(new_entry)
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
@@ -503,7 +487,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
||||
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
@@ -528,7 +512,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
||||
"""Update an existing memory item with new text and optional category."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
@@ -550,7 +534,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def delete_memory(request: Request, memory_id: str):
|
||||
"""Delete a memory item by its ID."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
|
||||
# Find and verify ownership before deleting
|
||||
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Vault route domain package (slice 2k, #4082/#4071).
|
||||
|
||||
Contains vault_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/vault_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,242 +0,0 @@
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
+237
-9
@@ -1,14 +1,242 @@
|
||||
"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
|
||||
and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
|
||||
by test_vault_password_not_in_argv.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2k (#4082/#4071).
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import sys as _sys
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from routes.vault import vault_routes as _canonical # noqa: F401
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Webhook route domain package (slice 2l, #4082/#4071).
|
||||
|
||||
Contains webhook_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/webhook_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,395 +0,0 @@
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
+391
-12
@@ -1,16 +1,395 @@
|
||||
"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
|
||||
``importlib.import_module("routes.webhook_routes")``, and the
|
||||
``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
|
||||
...)`` pattern used by test_null_owner_gates.py all operate on the *same*
|
||||
object. Keeps existing import paths working after slice 2l (#4082/#4071).
|
||||
Source-introspection tests read the canonical file by path.
|
||||
"""
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import sys as _sys
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from routes.webhook import webhook_routes as _canonical # noqa: F401
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Memory service — persistent memory storage and retrieval."""
|
||||
|
||||
from .service import MemoryService, Memory, MemorySearchResult
|
||||
from .memory import MemoryManager, MemoryStoreUnreadable
|
||||
from .memory import MemoryManager
|
||||
from .memory_vector import MemoryVectorStore
|
||||
|
||||
__all__ = [
|
||||
@@ -10,6 +10,5 @@ __all__ = [
|
||||
"Memory",
|
||||
"MemorySearchResult",
|
||||
"MemoryManager",
|
||||
"MemoryStoreUnreadable",
|
||||
"MemoryVectorStore",
|
||||
]
|
||||
|
||||
@@ -5,16 +5,6 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a
|
||||
parallel implementation here risks silent drift between import paths.
|
||||
"""
|
||||
|
||||
from src.memory import (
|
||||
MemoryManager,
|
||||
MemoryStoreUnreadable,
|
||||
get_text_similarity,
|
||||
tokenize,
|
||||
)
|
||||
from src.memory import MemoryManager, get_text_similarity, tokenize
|
||||
|
||||
__all__ = [
|
||||
"MemoryManager",
|
||||
"MemoryStoreUnreadable",
|
||||
"get_text_similarity",
|
||||
"tokenize",
|
||||
]
|
||||
__all__ = ["MemoryManager", "get_text_similarity", "tokenize"]
|
||||
|
||||
@@ -17,8 +17,6 @@ import os
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -389,13 +387,7 @@ async def extract_and_store(
|
||||
# Get owner from session
|
||||
_owner = getattr(session, 'owner', None)
|
||||
|
||||
# Strict load: this is a read-modify-write. Degrading to [] here would
|
||||
# save only the newly extracted facts and drop the entire store.
|
||||
try:
|
||||
existing = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Skipping auto memory extraction, store unreadable: %s", e)
|
||||
return
|
||||
existing = memory_manager.load_all()
|
||||
added = 0
|
||||
|
||||
for fact in facts:
|
||||
@@ -634,18 +626,7 @@ async def audit_memories(
|
||||
|
||||
# Merge audited entries back with other users' entries
|
||||
if owner:
|
||||
# Strict load: the merge below reconstructs the whole file. If this
|
||||
# degraded to [] we would save only this owner's audited slice and
|
||||
# destroy every other tenant's memories.
|
||||
try:
|
||||
all_entries = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Aborting memory audit save, store unreadable: %s", e)
|
||||
return {
|
||||
"before": before_count,
|
||||
"after": before_count,
|
||||
"error": "store_unreadable",
|
||||
}
|
||||
all_entries = memory_manager.load_all()
|
||||
audited_ids = {e["id"] for e in final_entries}
|
||||
other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
|
||||
# Also keep legacy entries that weren't part of this audit
|
||||
|
||||
+1
-10
@@ -22,7 +22,6 @@ import time
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
|
||||
|
||||
from src.constants import GENERATED_IMAGES_DIR
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -385,15 +384,7 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
|
||||
return {"error": "Memory text cannot be empty"}
|
||||
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||
# Strict load: this is a read-modify-write, and it is the path an
|
||||
# ordinary "remember that I prefer X" takes. Degrading to [] here would
|
||||
# save just this one entry over a store we only failed to read,
|
||||
# atomically destroying every memory in it (issue #5673).
|
||||
try:
|
||||
memories = _memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to add memory, store unreadable: %s", e)
|
||||
return {"error": "Memory store is temporarily unreadable — nothing was saved."}
|
||||
memories = _memory_manager.load_all()
|
||||
memories.append(entry)
|
||||
_memory_manager.save(memories)
|
||||
|
||||
|
||||
+3
-172
@@ -1,14 +1,11 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, List, Optional, Any
|
||||
from urllib.parse import urljoin, urlparse, urlunparse
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
|
||||
@@ -357,152 +354,6 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]:
|
||||
return None
|
||||
|
||||
|
||||
# httpcore raises its own exception hierarchy; map the ones a simple request can
|
||||
# surface back to their httpx equivalents so the caller's `except httpx.*` blocks
|
||||
# below behave exactly as they did with the default transport.
|
||||
_HTTPCORE_TO_HTTPX_EXC = {
|
||||
httpcore.ConnectError: httpx.ConnectError,
|
||||
httpcore.ConnectTimeout: httpx.ConnectTimeout,
|
||||
httpcore.NetworkError: httpx.NetworkError,
|
||||
httpcore.PoolTimeout: httpx.PoolTimeout,
|
||||
httpcore.ProtocolError: httpx.ProtocolError,
|
||||
httpcore.ReadError: httpx.ReadError,
|
||||
httpcore.ReadTimeout: httpx.ReadTimeout,
|
||||
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
|
||||
httpcore.TimeoutException: httpx.TimeoutException,
|
||||
httpcore.WriteError: httpx.WriteError,
|
||||
httpcore.WriteTimeout: httpx.WriteTimeout,
|
||||
}
|
||||
|
||||
|
||||
class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
|
||||
"""Network backend that connects only to the pre-validated IPs, in order.
|
||||
|
||||
Every address here came out of the single SSRF resolution, so moving to the
|
||||
next one after a connect failure is not re-resolution — it's ordinary
|
||||
multi-address fallback restricted to the set the guard already approved.
|
||||
httpcore takes TLS SNI and the ``Host`` header from the request URL rather
|
||||
than the connect host, so pinning the socket destination leaves certificate
|
||||
validation and vhost routing pointed at the original hostname.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._ips = [str(ip) for ip in ips]
|
||||
self._real = httpcore.AnyIOBackend()
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
# One shared connect budget: each attempt gets the time left until the
|
||||
# original deadline, so N dead addresses can't stretch the connect
|
||||
# phase to N * timeout.
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
last_exc: Optional[Exception] = None
|
||||
for ip in self._ips:
|
||||
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
|
||||
try:
|
||||
return await self._real.connect_tcp(
|
||||
ip, port, remaining, local_address, socket_options
|
||||
)
|
||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
|
||||
last_exc = exc
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
raise last_exc
|
||||
|
||||
async def connect_unix_socket(self, path, timeout=None, socket_options=None):
|
||||
return await self._real.connect_unix_socket(path, timeout, socket_options)
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
return await self._real.sleep(seconds)
|
||||
|
||||
|
||||
class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
|
||||
"""httpx transport that pins the TCP connect to the pre-resolved IP(s).
|
||||
|
||||
Kept local, mirroring the per-module pinned transports web fetch and
|
||||
webhook delivery already carry, rather than coupling api_call to the
|
||||
webhook subsystem. The request URL passes through unchanged, so SNI and the
|
||||
``Host`` header stay the original hostname; only the socket destination is
|
||||
pinned, which is what closes the rebinding window.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._pinned_ips = list(ips)
|
||||
self._pool = httpcore.AsyncConnectionPool(
|
||||
# Reuse the CA trust the default httpx client would build (certifi
|
||||
# plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so
|
||||
# swapping in this transport doesn't quietly change which chains
|
||||
# verify. ssl.create_default_context() would use system roots.
|
||||
ssl_context=httpx.create_ssl_context(),
|
||||
http1=True,
|
||||
http2=False,
|
||||
network_backend=_PinnedAsyncBackend(ips),
|
||||
)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
core_req = httpcore.Request(
|
||||
method=request.method,
|
||||
url=httpcore.URL(
|
||||
scheme=request.url.raw_scheme,
|
||||
host=request.url.raw_host,
|
||||
port=request.url.port,
|
||||
target=request.url.raw_path,
|
||||
),
|
||||
headers=request.headers.raw,
|
||||
content=request.stream,
|
||||
extensions=request.extensions,
|
||||
)
|
||||
try:
|
||||
core_resp = await self._pool.handle_async_request(core_req)
|
||||
content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
|
||||
await core_resp.aclose()
|
||||
except Exception as exc:
|
||||
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
|
||||
if mapped is not None:
|
||||
raise mapped(str(exc)) from exc
|
||||
raise
|
||||
return httpx.Response(
|
||||
status_code=core_resp.status,
|
||||
headers=core_resp.headers,
|
||||
content=content,
|
||||
extensions=core_resp.extensions,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._pool.aclose()
|
||||
|
||||
|
||||
def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
|
||||
"""Return every entry that parses as an IP address, de-duplicated, order
|
||||
preserved.
|
||||
|
||||
check_outbound_url only reports ok when *all* of these classify as safe, so
|
||||
the whole list is guard-approved and any of them is a legitimate connect
|
||||
target. Skipping unparseable entries mirrors how the guard walks the same
|
||||
resolver output.
|
||||
|
||||
De-duplication matters because the resolver is getaddrinfo(host, None) with
|
||||
no socktype filter, so glibc reports the same address once per socktype
|
||||
(SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three
|
||||
times. Without this, the connect fallback would spend the shared deadline
|
||||
retrying one dead address instead of moving on to a genuinely different one.
|
||||
"""
|
||||
ips: List[ipaddress._BaseAddress] = []
|
||||
seen = set()
|
||||
for raw in raw_ips:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id
|
||||
except ValueError:
|
||||
continue
|
||||
if ip in seen:
|
||||
continue
|
||||
seen.add(ip)
|
||||
ips.append(ip)
|
||||
return ips
|
||||
|
||||
|
||||
async def execute_api_call(
|
||||
integration_id: str,
|
||||
method: str,
|
||||
@@ -558,31 +409,13 @@ async def execute_api_call(
|
||||
# loopback for locked-down deployments. Private stays allowed by default
|
||||
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
|
||||
# primary use case.
|
||||
from src.url_safety import check_outbound_url, _default_resolver
|
||||
from src.url_safety import check_outbound_url
|
||||
block_private = os.getenv(
|
||||
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
|
||||
).lower() == "true"
|
||||
# Resolve the host exactly once and remember the IPs the guard validated so
|
||||
# the request below can be pinned to them. check_outbound_url only reports
|
||||
# (ok, reason); a plain httpx client re-resolves the host at connect time,
|
||||
# which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a
|
||||
# public IP for the guard and then flips to 169.254.169.254 for the connect
|
||||
# would reach cloud metadata with the integration's auth headers attached.
|
||||
resolved_ips: List[str] = []
|
||||
|
||||
def _recording_resolver(host: str) -> List[str]:
|
||||
ips = _default_resolver(host)
|
||||
resolved_ips[:] = ips
|
||||
return ips
|
||||
|
||||
ok, reason = check_outbound_url(
|
||||
url, block_private=block_private, resolver=_recording_resolver
|
||||
)
|
||||
ok, reason = check_outbound_url(url, block_private=block_private)
|
||||
if not ok:
|
||||
return {"error": f"URL rejected: {reason}", "exit_code": 1}
|
||||
pinned_ips = _validated_ips(resolved_ips)
|
||||
if not pinned_ips:
|
||||
return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1}
|
||||
|
||||
method = method.upper()
|
||||
|
||||
@@ -622,9 +455,7 @@ async def execute_api_call(
|
||||
auth = httpx.BasicAuth(parts[0], parts[1])
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips)
|
||||
) as client:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
url,
|
||||
|
||||
+8
-78
@@ -10,18 +10,6 @@ from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryStoreUnreadable(RuntimeError):
|
||||
"""memory.json exists on disk but could not be read or parsed.
|
||||
|
||||
"The contents are unknown" is categorically different from "there are no
|
||||
memories". A read-modify-write caller that conflates the two appends to an
|
||||
empty view and then persists it, destroying the whole store — the writes
|
||||
are atomic, so the loss is durable. Raised by
|
||||
:meth:`MemoryManager.load_all_for_update` so those callers fail closed.
|
||||
"""
|
||||
|
||||
|
||||
def tokenize(text: str) -> List[str]:
|
||||
"""Simple tokenizer that splits on whitespace and removes punctuation."""
|
||||
return [word.strip('.,!?";') for word in text.split()]
|
||||
@@ -122,70 +110,22 @@ class MemoryManager:
|
||||
with open(self.memory_file, 'w', encoding='utf-8') as f:
|
||||
json.dump([], f, ensure_ascii=False, indent=2)
|
||||
|
||||
def _read_entries(self) -> List[Dict]:
|
||||
"""Parse the store, or raise :class:`MemoryStoreUnreadable`.
|
||||
|
||||
Returns ``[]`` only when the file genuinely does not exist. Every other
|
||||
failure mode raises, so callers can tell "no memories" apart from
|
||||
"couldn't read the memories".
|
||||
"""
|
||||
def load_all(self) -> List[Dict]:
|
||||
"""Load all memory entries from JSON file (unfiltered)."""
|
||||
if not os.path.exists(self.memory_file):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(self.memory_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except OSError as e:
|
||||
# PermissionError is an OSError (a scanner holding the file, a
|
||||
# permissions problem, bad media).
|
||||
raise MemoryStoreUnreadable(
|
||||
f"cannot read {self.memory_file}: {e}"
|
||||
) from e
|
||||
except json.JSONDecodeError as e:
|
||||
# This is the branch that actually destroyed stores: the file reads
|
||||
# back fine, so nothing stops the save that follows. A truncated
|
||||
# memory.json is reachable because core/database.py rewrites it with
|
||||
# a plain open(..,"w") + json.dump during migration.
|
||||
#
|
||||
# Preserved behaviour: a corrupt store still gets one shot at the
|
||||
# pre-JSON memory.txt migration. Only raise when that finds nothing,
|
||||
# so we never report "empty" for a store we simply failed to parse.
|
||||
legacy = self._migrate_from_legacy()
|
||||
if legacy:
|
||||
return legacy
|
||||
raise MemoryStoreUnreadable(
|
||||
f"{self.memory_file} is not valid JSON: {e}"
|
||||
) from e
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise MemoryStoreUnreadable(
|
||||
f"{self.memory_file} is not a JSON array (got {type(data).__name__})"
|
||||
)
|
||||
if isinstance(data, list):
|
||||
return self._validate_entries(data)
|
||||
|
||||
def load_all(self) -> List[Dict]:
|
||||
"""Load all memory entries from JSON file (unfiltered).
|
||||
|
||||
Lenient by design: this feeds display, search, and context-injection
|
||||
paths, so an unreadable store degrades to an empty list rather than
|
||||
breaking chat. Never build a value from this that you intend to save
|
||||
back — use :meth:`load_all_for_update` for that.
|
||||
"""
|
||||
try:
|
||||
return self._read_entries()
|
||||
except MemoryStoreUnreadable as e:
|
||||
except (json.JSONDecodeError, PermissionError) as e:
|
||||
logger.error("Error loading memory.json: %s", e)
|
||||
return self._migrate_from_legacy()
|
||||
|
||||
return []
|
||||
|
||||
def load_all_for_update(self) -> List[Dict]:
|
||||
"""Load for a read-modify-write cycle.
|
||||
|
||||
Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]``
|
||||
so a caller can never append to an empty view and persist it over a
|
||||
store that was only temporarily unreadable (issue #5673).
|
||||
"""
|
||||
return self._read_entries()
|
||||
|
||||
def load(self, owner: str = None) -> List[Dict]:
|
||||
"""Load memory entries, optionally filtered by owner."""
|
||||
entries = self.load_all()
|
||||
@@ -195,12 +135,7 @@ class MemoryManager:
|
||||
|
||||
def claim_ownerless(self, owner: str):
|
||||
"""Assign all ownerless memory entries to the given owner."""
|
||||
try:
|
||||
entries = self.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
# Skip the sweep rather than rewrite the store from an unknown view.
|
||||
logger.error("Skipping ownerless claim, memory store unreadable: %s", e)
|
||||
return
|
||||
entries = self.load_all()
|
||||
changed = False
|
||||
claimed = 0
|
||||
for entry in entries:
|
||||
@@ -300,12 +235,7 @@ class MemoryManager:
|
||||
if not ids:
|
||||
return
|
||||
id_set = set(ids)
|
||||
try:
|
||||
entries = self.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
# Best-effort counter; never worth rewriting the store blind.
|
||||
logger.error("Skipping uses bump, memory store unreadable: %s", e)
|
||||
return
|
||||
entries = self.load_all()
|
||||
changed = False
|
||||
for e in entries:
|
||||
if e.get("id") in id_set:
|
||||
|
||||
@@ -157,11 +157,7 @@ class NativeMemoryProvider(MemoryProvider):
|
||||
if metadata:
|
||||
entry["metadata"] = dict(metadata)
|
||||
|
||||
# Strict load: read-modify-write. `load_all` degrades an unreadable
|
||||
# store to [], which would save this single entry over everything
|
||||
# already stored (issue #5673). The provider API has no error channel,
|
||||
# so MemoryStoreUnreadable propagates to the caller.
|
||||
memories = self.memory_manager.load_all_for_update()
|
||||
memories = self.memory_manager.load_all()
|
||||
memories.append(entry)
|
||||
self.memory_manager.save(memories)
|
||||
|
||||
@@ -227,10 +223,7 @@ class NativeMemoryProvider(MemoryProvider):
|
||||
]
|
||||
|
||||
async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool:
|
||||
# Strict load for the same reason: `remaining` is derived from this
|
||||
# list and saved back, so it must never be built from a store we
|
||||
# failed to read.
|
||||
memories = self.memory_manager.load_all_for_update()
|
||||
memories = self.memory_manager.load_all()
|
||||
remaining = []
|
||||
deleted_id = None
|
||||
|
||||
|
||||
+2
-4
@@ -46,9 +46,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
except ValueError:
|
||||
return {"error": "Invalid JSON arguments", "exit_code": 1}
|
||||
|
||||
action = (args.get("action") or "").strip().lower()
|
||||
if not action:
|
||||
return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1}
|
||||
action = (args.get("action") or "").lower()
|
||||
from services.memory.skills import SkillsManager
|
||||
from services.memory.skill_format import Skill, slugify
|
||||
from src.constants import DATA_DIR
|
||||
@@ -57,7 +55,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
# Accept legacy `skill_id` as an alias for `name`.
|
||||
name = (args.get("name") or args.get("skill_id") or "").strip()
|
||||
|
||||
if action in ("list", "index"):
|
||||
if action in ("list", "index", ""):
|
||||
all_skills = sm.load(owner=owner)
|
||||
if not all_skills:
|
||||
return {"results": "No skills yet. Create one with action='add'."}
|
||||
|
||||
@@ -76,7 +76,7 @@ def _load_webhook_routes_for_test(monkeypatch):
|
||||
module_name = "routes.webhook_routes_under_test"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name,
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook" / "webhook_routes.py",
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook_routes.py",
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
@@ -27,9 +27,6 @@ def _setup(monkeypatch, store, user="alice"):
|
||||
|
||||
mem = MagicMock()
|
||||
mem.load_all.return_value = list(store)
|
||||
# import_data reads through the strict loader so a store it cannot read is
|
||||
# never overwritten (#5673); the double has to offer the same entry point.
|
||||
mem.load_all_for_update.return_value = list(store)
|
||||
saved = {}
|
||||
mem.save.side_effect = lambda entries: saved.__setitem__("entries", entries)
|
||||
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Regression test for the document route shim (slice 2m, #4082/#4071).
|
||||
|
||||
The backward-compat shims at ``routes/document_routes.py`` and
|
||||
``routes/document_helpers.py`` use ``sys.modules`` replacement so the legacy
|
||||
import paths and the canonical ``routes.document.*`` paths resolve to the
|
||||
*same* module objects. This is required because multiple tests do
|
||||
``import routes.document_routes as droutes`` followed by
|
||||
``droutes.SessionLocal = ...`` / ``monkeypatch.setattr(droutes, ...)`` and
|
||||
``sys.modules.pop("routes.document_helpers")`` + re-import — for those to
|
||||
take effect at runtime, the legacy and canonical module objects must be
|
||||
identical.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.document_routes as _shim_routes # noqa: F401
|
||||
import routes.document_helpers as _shim_helpers # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_routes_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_routes")
|
||||
canonical = importlib.import_module("routes.document.document_routes")
|
||||
assert legacy is canonical
|
||||
|
||||
|
||||
def test_legacy_and_canonical_helpers_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_helpers")
|
||||
canonical = importlib.import_module("routes.document.document_helpers")
|
||||
assert legacy is canonical
|
||||
@@ -87,7 +87,7 @@ def test_known_imap_mailbox_call_sites_are_quoted():
|
||||
assert "conn.select(sent_name" not in pollers
|
||||
assert "imap.append(sent_folder" not in pollers
|
||||
|
||||
document_routes = Path("routes/document/document_routes.py").read_text()
|
||||
document_routes = Path("routes/document_routes.py").read_text()
|
||||
assert "conn.select(doc.source_email_folder" not in document_routes
|
||||
|
||||
|
||||
|
||||
@@ -9,13 +9,8 @@ link-local/metadata is always rejected; RFC-1918/loopback only when
|
||||
INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary
|
||||
use case, so private stays allowed by default).
|
||||
"""
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import ssl
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src import integrations
|
||||
@@ -102,238 +97,3 @@ async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch
|
||||
assert result["exit_code"] == 1
|
||||
assert "rejected" in result["error"].lower()
|
||||
client.request.assert_not_called()
|
||||
|
||||
|
||||
async def _call_capturing_transport(base_url, path="/items"):
|
||||
"""Drive execute_api_call and return (result, transport) where transport is
|
||||
the object passed to httpx.AsyncClient(transport=...)."""
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.headers = {"content-type": "application/json"}
|
||||
resp.json.return_value = {"ok": True}
|
||||
resp.text = '{"ok": true}'
|
||||
|
||||
client = AsyncMock()
|
||||
client.__aenter__ = AsyncMock(return_value=client)
|
||||
client.__aexit__ = AsyncMock(return_value=None)
|
||||
client.request = AsyncMock(return_value=resp)
|
||||
|
||||
captured = {}
|
||||
|
||||
def _fake_async_client(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return client
|
||||
|
||||
with (
|
||||
patch.object(integrations, "_find_integration",
|
||||
return_value=_integration(base_url)),
|
||||
patch("httpx.AsyncClient", side_effect=_fake_async_client),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, captured.get("transport"), client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connection_is_pinned_to_the_validated_ip(monkeypatch):
|
||||
"""DNS-rebinding defense: the guard resolves the host once to a benign
|
||||
public IP, and the request must be pinned to *that* IP so a host that
|
||||
rebinds to the metadata range at connect time can't be reached with the
|
||||
integration's auth headers. Static resolution passing the guard is not
|
||||
enough — a plain client would re-resolve at connect."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34"])
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://rebinding.attacker.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
client.request.assert_called_once()
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pin_carries_the_whole_validated_ip_set(monkeypatch):
|
||||
"""When a host resolves to several records the transport keeps all of them
|
||||
(check_outbound_url validated every one), in resolver order, so it can fall
|
||||
back past a dead first address instead of failing the whole call."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34", "198.51.100.7"])
|
||||
result, transport, _ = await _call_capturing_transport("http://multi.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34", "198.51.100.7"]
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
"""Stand-in for the connected socket the real backend returns."""
|
||||
|
||||
|
||||
class _RecordingBackend:
|
||||
"""Fake httpcore backend: connect_tcp fails for the addresses in `dead`
|
||||
and succeeds for the rest, recording the order it was asked to connect."""
|
||||
|
||||
def __init__(self, dead):
|
||||
self.dead = set(dead)
|
||||
self.attempts = []
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
self.attempts.append((host, timeout))
|
||||
if host in self.dead:
|
||||
raise httpcore.ConnectError(f"connection refused: {host}")
|
||||
return _FakeStream()
|
||||
|
||||
|
||||
def _pinned_backend(ips, dead):
|
||||
"""A _PinnedAsyncBackend whose underlying connect is the recording fake."""
|
||||
backend = integrations._PinnedAsyncBackend(ips)
|
||||
backend._real = _RecordingBackend(dead)
|
||||
return backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_falls_back_from_dead_first_to_live_second():
|
||||
"""first-dead / second-live: the pinned backend must try the next validated
|
||||
address when the first refuses, rather than surfacing the failure. It also
|
||||
ignores the `host` httpcore passes (the original hostname) and connects to
|
||||
the pinned IPs, which is what keeps TLS SNI / Host on the real hostname."""
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10"})
|
||||
|
||||
stream = await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
|
||||
assert isinstance(stream, _FakeStream)
|
||||
# Tried the dead address first, then the live one — never the hostname.
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
# Fallback shared one budget: the second attempt got the time left, not a fresh 5s.
|
||||
assert backend._real.attempts[1][1] <= 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_raises_when_every_validated_address_is_dead():
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10", "198.51.100.7"})
|
||||
|
||||
with pytest.raises(httpcore.ConnectError):
|
||||
await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_transport_reuses_httpx_ca_trust(monkeypatch):
|
||||
"""TLS trust must come from the same builder the default httpx client uses
|
||||
(certifi + SSL_CERT_FILE / SSL_CERT_DIR via trust_env), not from
|
||||
ssl.create_default_context()'s system roots — otherwise chains that verified
|
||||
under the old default client can silently stop verifying."""
|
||||
sentinel = ssl.create_default_context()
|
||||
calls = []
|
||||
|
||||
def _fake_create(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(httpx, "create_ssl_context", _fake_create)
|
||||
transport = integrations._PinnedAsyncTransport([ipaddress.ip_address("93.184.216.34")])
|
||||
try:
|
||||
assert calls, "transport did not build its context via httpx.create_ssl_context"
|
||||
assert transport._pool._ssl_context is sentinel
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_socket_falls_back_from_dead_first_to_live_second():
|
||||
"""End-to-end over real loopback sockets: pin [127.0.0.2 (nothing
|
||||
listening), 127.0.0.1 (live)], and the request must succeed by falling back
|
||||
to the second address while the Host header stays the original hostname —
|
||||
i.e. only the socket destination moved, vhost/SNI routing did not."""
|
||||
captured = {}
|
||||
|
||||
async def handle(reader, writer):
|
||||
request = await reader.read(4096)
|
||||
for line in request.split(b"\r\n"):
|
||||
if line.lower().startswith(b"host:"):
|
||||
captured["host"] = line.split(b":", 1)[1].strip().decode()
|
||||
writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
async with server:
|
||||
await server.start_serving()
|
||||
transport = integrations._PinnedAsyncTransport(
|
||||
[ipaddress.ip_address("127.0.0.2"), ipaddress.ip_address("127.0.0.1")]
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
resp = await client.get(f"http://pinned.example:{port}/health")
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.text == "hi"
|
||||
assert captured.get("host") == f"pinned.example:{port}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ip_literal_base_url_still_pins_and_is_not_rejected():
|
||||
"""A base_url that is already an IP has nothing to rebind, but it must not
|
||||
trip the "did not resolve" guard either.
|
||||
|
||||
check_outbound_url resolves even a literal (getaddrinfo returns the address
|
||||
itself), so the captured list is populated and the pin is a no-op rather
|
||||
than a rejection. Uses the real resolver on purpose — no monkeypatch — so
|
||||
this would catch the fail-closed branch firing on a literal.
|
||||
"""
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://93.184.216.34")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ipv6_base_url_pins_every_validated_address(monkeypatch):
|
||||
"""IPv6 goes down the same path as v4.
|
||||
|
||||
Resolution is stubbed rather than using a literal so this doesn't depend on
|
||||
the runner having IPv6 configured.
|
||||
"""
|
||||
v6 = "2606:2800:220:1:248:1893:25c8:1946"
|
||||
monkeypatch.setattr("src.url_safety._default_resolver", lambda host: [v6])
|
||||
result, transport, client = await _call_capturing_transport("http://v6.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == [v6]
|
||||
|
||||
|
||||
def test_validated_ips_strips_zone_id_and_drops_junk():
|
||||
"""getaddrinfo can hand back a scoped v6 address like 'fe80::1%eth0'."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "fe80::1%eth0", "not-an-ip", None, "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34", "fe80::1", "2001:db8::5"]
|
||||
|
||||
|
||||
def test_validated_ips_deduplicates_repeated_addresses():
|
||||
"""The resolver is getaddrinfo(host, None) with no socktype filter, so glibc
|
||||
returns one record per socktype and a single-homed host arrives three times
|
||||
over. Duplicates must collapse (first-seen order kept) or the connect
|
||||
fallback wastes its shared deadline retrying one dead address."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "93.184.216.34", "93.184.216.34"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34"]
|
||||
|
||||
# Order is first-seen, and distinct addresses all survive.
|
||||
got = integrations._validated_ips(
|
||||
["198.51.100.7", "93.184.216.34", "198.51.100.7", "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["198.51.100.7", "93.184.216.34", "2001:db8::5"]
|
||||
|
||||
# A zone-id variant is the same address once stripped, so it collapses too.
|
||||
got = integrations._validated_ips(["fe80::1%eth0", "fe80::1%eth1", "fe80::1"])
|
||||
assert [str(ip) for ip in got] == ["fe80::1"]
|
||||
|
||||
@@ -83,10 +83,9 @@ async def _call(json_data, status=200):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about truncation, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
return await integrations.execute_api_call("test_integ", "GET", "/items")
|
||||
|
||||
@@ -102,10 +101,9 @@ async def _call_with_integration(integration, path="/items"):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=integration),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about URL joining, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, mock_client
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.tools.system import do_manage_skills
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{},
|
||||
{"action": ""},
|
||||
{"action": " "},
|
||||
{"name": "demo", "description": "x", "procedure": ["step"]},
|
||||
],
|
||||
)
|
||||
async def test_manage_skills_requires_action(payload):
|
||||
result = await do_manage_skills(json.dumps(payload), owner="test")
|
||||
|
||||
assert result == {
|
||||
"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)",
|
||||
"exit_code": 1,
|
||||
}
|
||||
@@ -67,12 +67,6 @@ class FakeMemoryManager:
|
||||
def load_all(self):
|
||||
return list(self.rows)
|
||||
|
||||
def load_all_for_update(self):
|
||||
# Mirrors the real MemoryManager: extraction is a read-modify-write and
|
||||
# goes through the strict loader (#5673). A healthy store behaves the
|
||||
# same as load_all.
|
||||
return list(self.rows)
|
||||
|
||||
def load(self, owner=None):
|
||||
return [r for r in self.rows if r.get("owner") == owner]
|
||||
|
||||
|
||||
@@ -1,255 +0,0 @@
|
||||
"""A memory store that cannot be READ must never be overwritten (issue #5673).
|
||||
|
||||
`MemoryManager.save` is atomic, and the add/import/extract paths are all
|
||||
read-modify-write: load the whole store, append, save it back. `load_all`
|
||||
used to answer a *failed read* with `[]` — indistinguishable from "no
|
||||
memories" — so a failed read turned into
|
||||
|
||||
load_all() -> [] -> [].append(new) -> save([new])
|
||||
|
||||
which atomically replaced the entire store with one entry.
|
||||
|
||||
The trigger that actually bites is a store that is **readable but not
|
||||
parseable** — a truncated file, or one holding `{}` instead of `[]`. Nothing
|
||||
obstructs the write, so the request succeeds with HTTP 200 and every existing
|
||||
memory is destroyed silently. Truncation is reachable: `core/database.py`
|
||||
rewrites memory.json during migration with a plain `open(..., "w")` +
|
||||
`json.dump`, which is not atomic.
|
||||
|
||||
A live exclusive lock is NOT the dangerous case: it blocks the read and the
|
||||
`os.replace` alike, so the save fails too and the store survives (verified
|
||||
end-to-end — clean dev returns 500 there and loses nothing).
|
||||
|
||||
`load_all_for_update` is the strict loader those callers now use: it raises
|
||||
`MemoryStoreUnreadable` rather than reporting an empty store.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import builtins
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from src.memory import MemoryManager, MemoryStoreUnreadable
|
||||
|
||||
_SEED = [
|
||||
{"id": "m1", "text": "user prefers dark mode", "owner": "alice"},
|
||||
{"id": "m2", "text": "user lives in Berlin", "owner": "alice"},
|
||||
{"id": "m3", "text": "bob's cat is called Mila", "owner": "bob"},
|
||||
]
|
||||
|
||||
|
||||
def _seeded(tmp_path):
|
||||
m = MemoryManager(str(tmp_path))
|
||||
m.save([dict(e) for e in _SEED])
|
||||
return m
|
||||
|
||||
|
||||
def _break_reads_of(monkeypatch, target, exc):
|
||||
"""Make open() raise `exc` for `target` only, leaving every other path alone."""
|
||||
real_open = builtins.open
|
||||
|
||||
def fake_open(file, mode="r", *args, **kwargs):
|
||||
if os.path.abspath(str(file)) == os.path.abspath(target) and "r" in mode:
|
||||
raise exc
|
||||
return real_open(file, mode, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "open", fake_open)
|
||||
|
||||
|
||||
# ── the strict loader signals, rather than reporting "empty" ──────────────
|
||||
|
||||
def test_strict_load_raises_on_permission_error(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_raises_on_corrupt_json(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write('[{"id": "m1", "text": "truncated mid-writ')
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_raises_when_store_is_not_a_list(tmp_path):
|
||||
# A file holding `{}` or `null` is not an empty store, it is a broken one.
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
json.dump({}, f)
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_returns_entries_when_healthy(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
assert {e["id"] for e in m.load_all_for_update()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_strict_load_returns_empty_when_file_genuinely_absent(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
os.remove(m.memory_file)
|
||||
# Absent is the one case that legitimately means "no memories yet".
|
||||
assert m.load_all_for_update() == []
|
||||
|
||||
|
||||
# ── read paths stay lenient, so an unreadable store can't break chat ──────
|
||||
|
||||
def test_read_path_still_degrades_to_empty(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
||||
# Context injection / search must not raise; they just see nothing.
|
||||
assert m.load_all() == []
|
||||
assert m.load(owner="alice") == []
|
||||
|
||||
|
||||
# ── the actual #5673 regression: the store survives ───────────────────────
|
||||
|
||||
def test_add_cycle_under_transient_read_error_does_not_wipe(tmp_path, monkeypatch):
|
||||
"""Mirrors routes/memory/memory_routes.py api_add_memory exactly."""
|
||||
m = _seeded(tmp_path)
|
||||
new_entry = m.add_entry("a brand new fact", owner="alice")
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
all_mem = m.load_all_for_update()
|
||||
all_mem.append(new_entry)
|
||||
m.save(all_mem)
|
||||
|
||||
# Reads work again; every original memory is still there and the file was
|
||||
# never replaced by the single new entry.
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_audit_merge_cannot_drop_other_tenants(tmp_path, monkeypatch):
|
||||
"""The audit path rebuilds the whole file from load_all + one owner's slice.
|
||||
|
||||
Reading [] there would save only the audited owner's entries and destroy
|
||||
every other tenant's memories, so it has to fail closed too.
|
||||
"""
|
||||
m = _seeded(tmp_path)
|
||||
alice_slice = [e for e in _SEED if e["owner"] == "alice"]
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
all_entries = m.load_all_for_update()
|
||||
others = [e for e in all_entries if e.get("owner") != "alice"]
|
||||
m.save(alice_slice + others)
|
||||
|
||||
assert any(e["id"] == "m3" for e in m.load_all()), "bob's memory was destroyed"
|
||||
|
||||
|
||||
def test_uses_bump_skips_write_when_unreadable(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
m.increment_uses(["m1"]) # must not raise, must not write
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_claim_ownerless_skips_write_when_unreadable(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
m.claim_ownerless("alice")
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
# ── the add sinks users actually reach ────────────────────────────────────
|
||||
#
|
||||
# The tests above replay the read-modify-write shape. These drive the real
|
||||
# entry points end to end, because those are what #5673 reports: "remember
|
||||
# that I prefer X" in ordinary chat (src/ai_interaction.py do_manage_memory,
|
||||
# routed from src/tool_execution.py) and the built-in memory MCP server
|
||||
# (mcp_servers/memory_server.py, registered in src/builtin_mcp.py).
|
||||
#
|
||||
# They use a truncated store rather than a read error on purpose: it reads
|
||||
# fine, so nothing stops the save, which is the case that silently destroyed
|
||||
# stores. The assertion is that the file is left byte-identical — still broken,
|
||||
# but still holding the user's memories, so it can be repaired by hand.
|
||||
|
||||
|
||||
def _truncated_store(tmp_path):
|
||||
"""Seed a store that reads back fine but no longer parses."""
|
||||
m = _seeded(tmp_path)
|
||||
good = json.dumps([dict(e) for e in _SEED], indent=2)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write(good[:good.rindex("]")]) # drop the closing bracket only
|
||||
with open(m.memory_file, "rb") as f:
|
||||
return m, f.read()
|
||||
|
||||
|
||||
def _on_disk(manager) -> bytes:
|
||||
with open(manager.memory_file, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_agent_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
||||
"""src/ai_interaction.py do_manage_memory, action "add"."""
|
||||
from src import ai_interaction
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
monkeypatch.setattr(ai_interaction, "_memory_manager", manager)
|
||||
monkeypatch.setattr(ai_interaction, "_memory_vector", None)
|
||||
|
||||
result = asyncio.run(ai_interaction.do_manage_memory("add\nuser prefers tabs"))
|
||||
|
||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
||||
assert b"m3" in _on_disk(manager)
|
||||
assert "error" in result, "the add reported success over an unreadable store"
|
||||
|
||||
|
||||
def test_mcp_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
||||
"""mcp_servers/memory_server.py, action "add"."""
|
||||
import mcp_servers.memory_server as memory_server
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
monkeypatch.setattr(memory_server, "_memory_manager", manager)
|
||||
monkeypatch.setattr(memory_server, "_memory_vector", None)
|
||||
monkeypatch.setattr(memory_server, "_initialized", True)
|
||||
for key in memory_server._OWNER_ENV_KEYS:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
result = asyncio.run(memory_server.call_tool(
|
||||
"manage_memory", {"action": "add", "text": "user prefers tabs"}
|
||||
))
|
||||
|
||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
||||
assert b"m3" in _on_disk(manager)
|
||||
assert result[0].text.startswith("Error:")
|
||||
|
||||
|
||||
def test_native_provider_remember_does_not_overwrite_unreadable_store(tmp_path):
|
||||
"""src/memory_provider.py NativeMemoryProvider.remember.
|
||||
|
||||
Registered into app state in src/app_initializer.py but not yet consumed
|
||||
outside tests, so this is the pattern held in place before it goes live.
|
||||
"""
|
||||
from src.memory_provider import NativeMemoryProvider
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
provider = NativeMemoryProvider(manager)
|
||||
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
asyncio.run(provider.remember("user prefers tabs", owner="alice"))
|
||||
|
||||
assert _on_disk(manager) == before
|
||||
|
||||
|
||||
# ── the legacy memory.txt migration is preserved ──────────────────────────
|
||||
|
||||
def test_corrupt_store_still_migrates_from_legacy_txt(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write("{ not json")
|
||||
legacy = os.path.join(str(tmp_path), "memory.txt")
|
||||
with open(legacy, "w", encoding="utf-8") as f:
|
||||
f.write("recovered fact one\nrecovered fact two\n")
|
||||
|
||||
entries = m.load_all_for_update()
|
||||
assert [e["text"] for e in entries] == ["recovered fact one", "recovered fact two"]
|
||||
@@ -14,7 +14,7 @@ def _function_source(path: str, name: str) -> str:
|
||||
|
||||
|
||||
def test_document_ai_tidy_resolves_with_owner_scope():
|
||||
body = _function_source("routes/document/document_routes.py", "ai_tidy_documents")
|
||||
body = _function_source("routes/document_routes.py", "ai_tidy_documents")
|
||||
assert "resolve_task_endpoint(owner=user or None)" in body
|
||||
assert 'resolve_endpoint("default", owner=user or None)' in body
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the vault route shim (slice 2k, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.vault_routes as _shim_vault # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_vault_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.vault_routes")
|
||||
canonical = importlib.import_module("routes.vault.vault_routes")
|
||||
assert legacy is canonical
|
||||
@@ -88,7 +88,7 @@ def test_request_vision_call_sites_pass_owner():
|
||||
chat_source = (ROOT / "src" / "chat_handler.py").read_text()
|
||||
processor_source = (ROOT / "src" / "document_processor.py").read_text()
|
||||
upload_source = (ROOT / "routes" / "upload_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document" / "document_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document_routes.py").read_text()
|
||||
gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text()
|
||||
memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text()
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the webhook route shim (slice 2l, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.webhook_routes as _shim_webhook # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_webhook_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.webhook_routes")
|
||||
canonical = importlib.import_module("routes.webhook.webhook_routes")
|
||||
assert legacy is canonical
|
||||
Reference in New Issue
Block a user