mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-04 12:38:51 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
6b8c3ae6b0 |
@@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service))
|
||||
logger.info("STT service initialized (provider managed via settings)")
|
||||
|
||||
# Documents (artifacts/canvas)
|
||||
from routes.document.document_routes import setup_document_routes
|
||||
from routes.document_routes import setup_document_routes
|
||||
document_router = setup_document_routes(session_manager, upload_handler)
|
||||
app.include_router(document_router)
|
||||
|
||||
@@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
|
||||
logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
|
||||
|
||||
# Webhooks
|
||||
from routes.webhook.webhook_routes import setup_webhook_routes
|
||||
from routes.webhook_routes import setup_webhook_routes
|
||||
app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
|
||||
|
||||
# API Tokens
|
||||
@@ -852,7 +852,7 @@ app.include_router(setup_codex_routes(
|
||||
))
|
||||
app.include_router(setup_claude_routes())
|
||||
|
||||
from routes.vault.vault_routes import setup_vault_routes
|
||||
from routes.vault_routes import setup_vault_routes
|
||||
app.include_router(setup_vault_routes())
|
||||
|
||||
# Contacts (CardDAV)
|
||||
|
||||
@@ -33,4 +33,4 @@ PyMuPDF
|
||||
# magika (onnxruntime), already a core dep via fastembed. We avoid the
|
||||
# [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
|
||||
# the dependency-age discussion in issue #485.
|
||||
markitdown[docx,pptx,xlsx,xls]==0.1.6
|
||||
markitdown[docx,pptx,xlsx,xls]==0.1.7
|
||||
|
||||
+3
-3
@@ -3,9 +3,9 @@ uvicorn
|
||||
python-multipart
|
||||
python-dotenv
|
||||
httpx
|
||||
httpcore>=1.0,<2.0
|
||||
httpcore>=1.0.9,<2.0
|
||||
pydantic>=2.13.4
|
||||
pydantic-settings>=2.14.1
|
||||
pydantic-settings>=2.14.2
|
||||
SQLAlchemy
|
||||
pypdf
|
||||
beautifulsoup4
|
||||
@@ -41,7 +41,7 @@ bcrypt
|
||||
# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
|
||||
# breaking rewrite, so keep fresh installs on the maintained v1 line until the
|
||||
# servers are migrated together.
|
||||
mcp<2
|
||||
mcp<3
|
||||
pyotp
|
||||
qrcode[pil]
|
||||
croniter
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
"""Document route domain package (slice 2m, #4082/#4071).
|
||||
|
||||
Contains document_routes.py and document_helpers.py, migrated from the flat
|
||||
routes/ directory. Backward-compat shims at routes/document_routes.py and
|
||||
routes/document_helpers.py re-export from here.
|
||||
"""
|
||||
@@ -1,243 +0,0 @@
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
File diff suppressed because it is too large
Load Diff
+239
-10
@@ -1,14 +1,243 @@
|
||||
"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.document_helpers``, ``from routes.document_helpers import
|
||||
X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
|
||||
pattern used by test_security_regressions.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2m (#4082/#4071).
|
||||
"""
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import sys as _sys
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from routes.document import document_helpers as _canonical # noqa: F401
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
|
||||
+1806
-13
File diff suppressed because it is too large
Load Diff
@@ -1,5 +0,0 @@
|
||||
"""Vault route domain package (slice 2k, #4082/#4071).
|
||||
|
||||
Contains vault_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/vault_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,242 +0,0 @@
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
+237
-9
@@ -1,14 +1,242 @@
|
||||
"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
|
||||
and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
|
||||
by test_vault_password_not_in_argv.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2k (#4082/#4071).
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import sys as _sys
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from routes.vault import vault_routes as _canonical # noqa: F401
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Webhook route domain package (slice 2l, #4082/#4071).
|
||||
|
||||
Contains webhook_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/webhook_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,395 +0,0 @@
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
+391
-12
@@ -1,16 +1,395 @@
|
||||
"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
|
||||
``importlib.import_module("routes.webhook_routes")``, and the
|
||||
``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
|
||||
...)`` pattern used by test_null_owner_gates.py all operate on the *same*
|
||||
object. Keeps existing import paths working after slice 2l (#4082/#4071).
|
||||
Source-introspection tests read the canonical file by path.
|
||||
"""
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import sys as _sys
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from routes.webhook import webhook_routes as _canonical # noqa: F401
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
|
||||
+3
-172
@@ -1,14 +1,11 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, List, Optional, Any
|
||||
from urllib.parse import urljoin, urlparse, urlunparse
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
|
||||
@@ -357,152 +354,6 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]:
|
||||
return None
|
||||
|
||||
|
||||
# httpcore raises its own exception hierarchy; map the ones a simple request can
|
||||
# surface back to their httpx equivalents so the caller's `except httpx.*` blocks
|
||||
# below behave exactly as they did with the default transport.
|
||||
_HTTPCORE_TO_HTTPX_EXC = {
|
||||
httpcore.ConnectError: httpx.ConnectError,
|
||||
httpcore.ConnectTimeout: httpx.ConnectTimeout,
|
||||
httpcore.NetworkError: httpx.NetworkError,
|
||||
httpcore.PoolTimeout: httpx.PoolTimeout,
|
||||
httpcore.ProtocolError: httpx.ProtocolError,
|
||||
httpcore.ReadError: httpx.ReadError,
|
||||
httpcore.ReadTimeout: httpx.ReadTimeout,
|
||||
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
|
||||
httpcore.TimeoutException: httpx.TimeoutException,
|
||||
httpcore.WriteError: httpx.WriteError,
|
||||
httpcore.WriteTimeout: httpx.WriteTimeout,
|
||||
}
|
||||
|
||||
|
||||
class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
|
||||
"""Network backend that connects only to the pre-validated IPs, in order.
|
||||
|
||||
Every address here came out of the single SSRF resolution, so moving to the
|
||||
next one after a connect failure is not re-resolution — it's ordinary
|
||||
multi-address fallback restricted to the set the guard already approved.
|
||||
httpcore takes TLS SNI and the ``Host`` header from the request URL rather
|
||||
than the connect host, so pinning the socket destination leaves certificate
|
||||
validation and vhost routing pointed at the original hostname.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._ips = [str(ip) for ip in ips]
|
||||
self._real = httpcore.AnyIOBackend()
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
# One shared connect budget: each attempt gets the time left until the
|
||||
# original deadline, so N dead addresses can't stretch the connect
|
||||
# phase to N * timeout.
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
last_exc: Optional[Exception] = None
|
||||
for ip in self._ips:
|
||||
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
|
||||
try:
|
||||
return await self._real.connect_tcp(
|
||||
ip, port, remaining, local_address, socket_options
|
||||
)
|
||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
|
||||
last_exc = exc
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
raise last_exc
|
||||
|
||||
async def connect_unix_socket(self, path, timeout=None, socket_options=None):
|
||||
return await self._real.connect_unix_socket(path, timeout, socket_options)
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
return await self._real.sleep(seconds)
|
||||
|
||||
|
||||
class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
|
||||
"""httpx transport that pins the TCP connect to the pre-resolved IP(s).
|
||||
|
||||
Kept local, mirroring the per-module pinned transports web fetch and
|
||||
webhook delivery already carry, rather than coupling api_call to the
|
||||
webhook subsystem. The request URL passes through unchanged, so SNI and the
|
||||
``Host`` header stay the original hostname; only the socket destination is
|
||||
pinned, which is what closes the rebinding window.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._pinned_ips = list(ips)
|
||||
self._pool = httpcore.AsyncConnectionPool(
|
||||
# Reuse the CA trust the default httpx client would build (certifi
|
||||
# plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so
|
||||
# swapping in this transport doesn't quietly change which chains
|
||||
# verify. ssl.create_default_context() would use system roots.
|
||||
ssl_context=httpx.create_ssl_context(),
|
||||
http1=True,
|
||||
http2=False,
|
||||
network_backend=_PinnedAsyncBackend(ips),
|
||||
)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
core_req = httpcore.Request(
|
||||
method=request.method,
|
||||
url=httpcore.URL(
|
||||
scheme=request.url.raw_scheme,
|
||||
host=request.url.raw_host,
|
||||
port=request.url.port,
|
||||
target=request.url.raw_path,
|
||||
),
|
||||
headers=request.headers.raw,
|
||||
content=request.stream,
|
||||
extensions=request.extensions,
|
||||
)
|
||||
try:
|
||||
core_resp = await self._pool.handle_async_request(core_req)
|
||||
content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
|
||||
await core_resp.aclose()
|
||||
except Exception as exc:
|
||||
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
|
||||
if mapped is not None:
|
||||
raise mapped(str(exc)) from exc
|
||||
raise
|
||||
return httpx.Response(
|
||||
status_code=core_resp.status,
|
||||
headers=core_resp.headers,
|
||||
content=content,
|
||||
extensions=core_resp.extensions,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._pool.aclose()
|
||||
|
||||
|
||||
def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
|
||||
"""Return every entry that parses as an IP address, de-duplicated, order
|
||||
preserved.
|
||||
|
||||
check_outbound_url only reports ok when *all* of these classify as safe, so
|
||||
the whole list is guard-approved and any of them is a legitimate connect
|
||||
target. Skipping unparseable entries mirrors how the guard walks the same
|
||||
resolver output.
|
||||
|
||||
De-duplication matters because the resolver is getaddrinfo(host, None) with
|
||||
no socktype filter, so glibc reports the same address once per socktype
|
||||
(SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three
|
||||
times. Without this, the connect fallback would spend the shared deadline
|
||||
retrying one dead address instead of moving on to a genuinely different one.
|
||||
"""
|
||||
ips: List[ipaddress._BaseAddress] = []
|
||||
seen = set()
|
||||
for raw in raw_ips:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id
|
||||
except ValueError:
|
||||
continue
|
||||
if ip in seen:
|
||||
continue
|
||||
seen.add(ip)
|
||||
ips.append(ip)
|
||||
return ips
|
||||
|
||||
|
||||
async def execute_api_call(
|
||||
integration_id: str,
|
||||
method: str,
|
||||
@@ -558,31 +409,13 @@ async def execute_api_call(
|
||||
# loopback for locked-down deployments. Private stays allowed by default
|
||||
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
|
||||
# primary use case.
|
||||
from src.url_safety import check_outbound_url, _default_resolver
|
||||
from src.url_safety import check_outbound_url
|
||||
block_private = os.getenv(
|
||||
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
|
||||
).lower() == "true"
|
||||
# Resolve the host exactly once and remember the IPs the guard validated so
|
||||
# the request below can be pinned to them. check_outbound_url only reports
|
||||
# (ok, reason); a plain httpx client re-resolves the host at connect time,
|
||||
# which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a
|
||||
# public IP for the guard and then flips to 169.254.169.254 for the connect
|
||||
# would reach cloud metadata with the integration's auth headers attached.
|
||||
resolved_ips: List[str] = []
|
||||
|
||||
def _recording_resolver(host: str) -> List[str]:
|
||||
ips = _default_resolver(host)
|
||||
resolved_ips[:] = ips
|
||||
return ips
|
||||
|
||||
ok, reason = check_outbound_url(
|
||||
url, block_private=block_private, resolver=_recording_resolver
|
||||
)
|
||||
ok, reason = check_outbound_url(url, block_private=block_private)
|
||||
if not ok:
|
||||
return {"error": f"URL rejected: {reason}", "exit_code": 1}
|
||||
pinned_ips = _validated_ips(resolved_ips)
|
||||
if not pinned_ips:
|
||||
return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1}
|
||||
|
||||
method = method.upper()
|
||||
|
||||
@@ -622,9 +455,7 @@ async def execute_api_call(
|
||||
auth = httpx.BasicAuth(parts[0], parts[1])
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips)
|
||||
) as client:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
url,
|
||||
|
||||
+2
-4
@@ -46,9 +46,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
except ValueError:
|
||||
return {"error": "Invalid JSON arguments", "exit_code": 1}
|
||||
|
||||
action = (args.get("action") or "").strip().lower()
|
||||
if not action:
|
||||
return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1}
|
||||
action = (args.get("action") or "").lower()
|
||||
from services.memory.skills import SkillsManager
|
||||
from services.memory.skill_format import Skill, slugify
|
||||
from src.constants import DATA_DIR
|
||||
@@ -57,7 +55,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
# Accept legacy `skill_id` as an alias for `name`.
|
||||
name = (args.get("name") or args.get("skill_id") or "").strip()
|
||||
|
||||
if action in ("list", "index"):
|
||||
if action in ("list", "index", ""):
|
||||
all_skills = sm.load(owner=owner)
|
||||
if not all_skills:
|
||||
return {"results": "No skills yet. Create one with action='add'."}
|
||||
|
||||
@@ -76,7 +76,7 @@ def _load_webhook_routes_for_test(monkeypatch):
|
||||
module_name = "routes.webhook_routes_under_test"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name,
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook" / "webhook_routes.py",
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook_routes.py",
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Regression test for the document route shim (slice 2m, #4082/#4071).
|
||||
|
||||
The backward-compat shims at ``routes/document_routes.py`` and
|
||||
``routes/document_helpers.py`` use ``sys.modules`` replacement so the legacy
|
||||
import paths and the canonical ``routes.document.*`` paths resolve to the
|
||||
*same* module objects. This is required because multiple tests do
|
||||
``import routes.document_routes as droutes`` followed by
|
||||
``droutes.SessionLocal = ...`` / ``monkeypatch.setattr(droutes, ...)`` and
|
||||
``sys.modules.pop("routes.document_helpers")`` + re-import — for those to
|
||||
take effect at runtime, the legacy and canonical module objects must be
|
||||
identical.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.document_routes as _shim_routes # noqa: F401
|
||||
import routes.document_helpers as _shim_helpers # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_routes_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_routes")
|
||||
canonical = importlib.import_module("routes.document.document_routes")
|
||||
assert legacy is canonical
|
||||
|
||||
|
||||
def test_legacy_and_canonical_helpers_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_helpers")
|
||||
canonical = importlib.import_module("routes.document.document_helpers")
|
||||
assert legacy is canonical
|
||||
@@ -87,7 +87,7 @@ def test_known_imap_mailbox_call_sites_are_quoted():
|
||||
assert "conn.select(sent_name" not in pollers
|
||||
assert "imap.append(sent_folder" not in pollers
|
||||
|
||||
document_routes = Path("routes/document/document_routes.py").read_text()
|
||||
document_routes = Path("routes/document_routes.py").read_text()
|
||||
assert "conn.select(doc.source_email_folder" not in document_routes
|
||||
|
||||
|
||||
|
||||
@@ -9,13 +9,8 @@ link-local/metadata is always rejected; RFC-1918/loopback only when
|
||||
INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary
|
||||
use case, so private stays allowed by default).
|
||||
"""
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import ssl
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src import integrations
|
||||
@@ -102,238 +97,3 @@ async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch
|
||||
assert result["exit_code"] == 1
|
||||
assert "rejected" in result["error"].lower()
|
||||
client.request.assert_not_called()
|
||||
|
||||
|
||||
async def _call_capturing_transport(base_url, path="/items"):
|
||||
"""Drive execute_api_call and return (result, transport) where transport is
|
||||
the object passed to httpx.AsyncClient(transport=...)."""
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.headers = {"content-type": "application/json"}
|
||||
resp.json.return_value = {"ok": True}
|
||||
resp.text = '{"ok": true}'
|
||||
|
||||
client = AsyncMock()
|
||||
client.__aenter__ = AsyncMock(return_value=client)
|
||||
client.__aexit__ = AsyncMock(return_value=None)
|
||||
client.request = AsyncMock(return_value=resp)
|
||||
|
||||
captured = {}
|
||||
|
||||
def _fake_async_client(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return client
|
||||
|
||||
with (
|
||||
patch.object(integrations, "_find_integration",
|
||||
return_value=_integration(base_url)),
|
||||
patch("httpx.AsyncClient", side_effect=_fake_async_client),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, captured.get("transport"), client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connection_is_pinned_to_the_validated_ip(monkeypatch):
|
||||
"""DNS-rebinding defense: the guard resolves the host once to a benign
|
||||
public IP, and the request must be pinned to *that* IP so a host that
|
||||
rebinds to the metadata range at connect time can't be reached with the
|
||||
integration's auth headers. Static resolution passing the guard is not
|
||||
enough — a plain client would re-resolve at connect."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34"])
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://rebinding.attacker.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
client.request.assert_called_once()
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pin_carries_the_whole_validated_ip_set(monkeypatch):
|
||||
"""When a host resolves to several records the transport keeps all of them
|
||||
(check_outbound_url validated every one), in resolver order, so it can fall
|
||||
back past a dead first address instead of failing the whole call."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34", "198.51.100.7"])
|
||||
result, transport, _ = await _call_capturing_transport("http://multi.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34", "198.51.100.7"]
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
"""Stand-in for the connected socket the real backend returns."""
|
||||
|
||||
|
||||
class _RecordingBackend:
|
||||
"""Fake httpcore backend: connect_tcp fails for the addresses in `dead`
|
||||
and succeeds for the rest, recording the order it was asked to connect."""
|
||||
|
||||
def __init__(self, dead):
|
||||
self.dead = set(dead)
|
||||
self.attempts = []
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
self.attempts.append((host, timeout))
|
||||
if host in self.dead:
|
||||
raise httpcore.ConnectError(f"connection refused: {host}")
|
||||
return _FakeStream()
|
||||
|
||||
|
||||
def _pinned_backend(ips, dead):
|
||||
"""A _PinnedAsyncBackend whose underlying connect is the recording fake."""
|
||||
backend = integrations._PinnedAsyncBackend(ips)
|
||||
backend._real = _RecordingBackend(dead)
|
||||
return backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_falls_back_from_dead_first_to_live_second():
|
||||
"""first-dead / second-live: the pinned backend must try the next validated
|
||||
address when the first refuses, rather than surfacing the failure. It also
|
||||
ignores the `host` httpcore passes (the original hostname) and connects to
|
||||
the pinned IPs, which is what keeps TLS SNI / Host on the real hostname."""
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10"})
|
||||
|
||||
stream = await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
|
||||
assert isinstance(stream, _FakeStream)
|
||||
# Tried the dead address first, then the live one — never the hostname.
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
# Fallback shared one budget: the second attempt got the time left, not a fresh 5s.
|
||||
assert backend._real.attempts[1][1] <= 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_raises_when_every_validated_address_is_dead():
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10", "198.51.100.7"})
|
||||
|
||||
with pytest.raises(httpcore.ConnectError):
|
||||
await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_transport_reuses_httpx_ca_trust(monkeypatch):
|
||||
"""TLS trust must come from the same builder the default httpx client uses
|
||||
(certifi + SSL_CERT_FILE / SSL_CERT_DIR via trust_env), not from
|
||||
ssl.create_default_context()'s system roots — otherwise chains that verified
|
||||
under the old default client can silently stop verifying."""
|
||||
sentinel = ssl.create_default_context()
|
||||
calls = []
|
||||
|
||||
def _fake_create(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(httpx, "create_ssl_context", _fake_create)
|
||||
transport = integrations._PinnedAsyncTransport([ipaddress.ip_address("93.184.216.34")])
|
||||
try:
|
||||
assert calls, "transport did not build its context via httpx.create_ssl_context"
|
||||
assert transport._pool._ssl_context is sentinel
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_socket_falls_back_from_dead_first_to_live_second():
|
||||
"""End-to-end over real loopback sockets: pin [127.0.0.2 (nothing
|
||||
listening), 127.0.0.1 (live)], and the request must succeed by falling back
|
||||
to the second address while the Host header stays the original hostname —
|
||||
i.e. only the socket destination moved, vhost/SNI routing did not."""
|
||||
captured = {}
|
||||
|
||||
async def handle(reader, writer):
|
||||
request = await reader.read(4096)
|
||||
for line in request.split(b"\r\n"):
|
||||
if line.lower().startswith(b"host:"):
|
||||
captured["host"] = line.split(b":", 1)[1].strip().decode()
|
||||
writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
async with server:
|
||||
await server.start_serving()
|
||||
transport = integrations._PinnedAsyncTransport(
|
||||
[ipaddress.ip_address("127.0.0.2"), ipaddress.ip_address("127.0.0.1")]
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
resp = await client.get(f"http://pinned.example:{port}/health")
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.text == "hi"
|
||||
assert captured.get("host") == f"pinned.example:{port}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ip_literal_base_url_still_pins_and_is_not_rejected():
|
||||
"""A base_url that is already an IP has nothing to rebind, but it must not
|
||||
trip the "did not resolve" guard either.
|
||||
|
||||
check_outbound_url resolves even a literal (getaddrinfo returns the address
|
||||
itself), so the captured list is populated and the pin is a no-op rather
|
||||
than a rejection. Uses the real resolver on purpose — no monkeypatch — so
|
||||
this would catch the fail-closed branch firing on a literal.
|
||||
"""
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://93.184.216.34")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ipv6_base_url_pins_every_validated_address(monkeypatch):
|
||||
"""IPv6 goes down the same path as v4.
|
||||
|
||||
Resolution is stubbed rather than using a literal so this doesn't depend on
|
||||
the runner having IPv6 configured.
|
||||
"""
|
||||
v6 = "2606:2800:220:1:248:1893:25c8:1946"
|
||||
monkeypatch.setattr("src.url_safety._default_resolver", lambda host: [v6])
|
||||
result, transport, client = await _call_capturing_transport("http://v6.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == [v6]
|
||||
|
||||
|
||||
def test_validated_ips_strips_zone_id_and_drops_junk():
|
||||
"""getaddrinfo can hand back a scoped v6 address like 'fe80::1%eth0'."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "fe80::1%eth0", "not-an-ip", None, "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34", "fe80::1", "2001:db8::5"]
|
||||
|
||||
|
||||
def test_validated_ips_deduplicates_repeated_addresses():
|
||||
"""The resolver is getaddrinfo(host, None) with no socktype filter, so glibc
|
||||
returns one record per socktype and a single-homed host arrives three times
|
||||
over. Duplicates must collapse (first-seen order kept) or the connect
|
||||
fallback wastes its shared deadline retrying one dead address."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "93.184.216.34", "93.184.216.34"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34"]
|
||||
|
||||
# Order is first-seen, and distinct addresses all survive.
|
||||
got = integrations._validated_ips(
|
||||
["198.51.100.7", "93.184.216.34", "198.51.100.7", "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["198.51.100.7", "93.184.216.34", "2001:db8::5"]
|
||||
|
||||
# A zone-id variant is the same address once stripped, so it collapses too.
|
||||
got = integrations._validated_ips(["fe80::1%eth0", "fe80::1%eth1", "fe80::1"])
|
||||
assert [str(ip) for ip in got] == ["fe80::1"]
|
||||
|
||||
@@ -83,10 +83,9 @@ async def _call(json_data, status=200):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about truncation, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
return await integrations.execute_api_call("test_integ", "GET", "/items")
|
||||
|
||||
@@ -102,10 +101,9 @@ async def _call_with_integration(integration, path="/items"):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=integration),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about URL joining, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, mock_client
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.tools.system import do_manage_skills
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{},
|
||||
{"action": ""},
|
||||
{"action": " "},
|
||||
{"name": "demo", "description": "x", "procedure": ["step"]},
|
||||
],
|
||||
)
|
||||
async def test_manage_skills_requires_action(payload):
|
||||
result = await do_manage_skills(json.dumps(payload), owner="test")
|
||||
|
||||
assert result == {
|
||||
"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)",
|
||||
"exit_code": 1,
|
||||
}
|
||||
@@ -14,7 +14,7 @@ def _function_source(path: str, name: str) -> str:
|
||||
|
||||
|
||||
def test_document_ai_tidy_resolves_with_owner_scope():
|
||||
body = _function_source("routes/document/document_routes.py", "ai_tidy_documents")
|
||||
body = _function_source("routes/document_routes.py", "ai_tidy_documents")
|
||||
assert "resolve_task_endpoint(owner=user or None)" in body
|
||||
assert 'resolve_endpoint("default", owner=user or None)' in body
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the vault route shim (slice 2k, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.vault_routes as _shim_vault # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_vault_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.vault_routes")
|
||||
canonical = importlib.import_module("routes.vault.vault_routes")
|
||||
assert legacy is canonical
|
||||
@@ -88,7 +88,7 @@ def test_request_vision_call_sites_pass_owner():
|
||||
chat_source = (ROOT / "src" / "chat_handler.py").read_text()
|
||||
processor_source = (ROOT / "src" / "document_processor.py").read_text()
|
||||
upload_source = (ROOT / "routes" / "upload_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document" / "document_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document_routes.py").read_text()
|
||||
gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text()
|
||||
memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text()
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the webhook route shim (slice 2l, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.webhook_routes as _shim_webhook # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_webhook_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.webhook_routes")
|
||||
canonical = importlib.import_module("routes.webhook.webhook_routes")
|
||||
assert legacy is canonical
|
||||
Reference in New Issue
Block a user